From 4d8df8f534f12500bc79fc5b00d300fa9e45a13d Mon Sep 17 00:00:00 2001 From: euzu <33094714+euzu@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:13:10 +0200 Subject: [PATCH] M3u catchup fix (#794) * **New Features** * Catch-up streams can now auto-detect whether to serve direct video or an HLS playlist. * Added archive-aware HLS session/identity handling and improved upstream user-agent propagation for manifests. * Enhanced M3U timeshift/archiving, including nested handling and native Flussonic archive support; Short EPG can advertise archive availability. * **Bug Fixes** * Provider session headers now persist across retries and deferred-open scenarios. * Legacy HLS routing is now cache-aware; seek detection improvements. * Stalker errors now better redact sensitive URL details. * Fix: Preserve VOD client range requests * **Tests** * Expanded coverage for probing, archive/timeshift resolution, header overlay rules, and related routing. --- CHANGELOG.md | 15 + backend/src/api/api_utils.rs | 384 +++++- .../src/api/endpoints/api_playlist_utils.rs | 2 +- backend/src/api/endpoints/hls_api.rs | 397 +++++- backend/src/api/endpoints/m3u_api.rs | 473 ++++++- backend/src/api/endpoints/v1_api.rs | 1 + backend/src/api/endpoints/v1_api_config.rs | 3 +- backend/src/api/endpoints/v1_api_playlist.rs | 4 +- backend/src/api/endpoints/xmltv_api.rs | 54 +- backend/src/api/endpoints/xtream_api.rs | 546 ++++++++- backend/src/api/model/active_user_manager.rs | 3 + backend/src/api/model/connection_manager.rs | 1 + backend/src/api/model/hls_cache/ids.rs | 38 +- backend/src/api/model/hls_cache/lease.rs | 28 + backend/src/api/model/hls_cache/origin.rs | 67 +- backend/src/api/model/hls_cache/playback.rs | 4 + .../src/api/model/metadata_update_manager.rs | 1 + backend/src/api/model/stream.rs | 9 +- .../api/model/streams/active_client_stream.rs | 19 +- .../model/streams/provider_stream_factory.rs | 222 ++-- .../model/streams/shared_stream_manager.rs | 1 + backend/src/api/panel_api.rs | 2 + backend/src/api/serve.rs | 1 + backend/src/iptv/m3u/archive.rs | 293 +++++ backend/src/iptv/m3u/catchup.rs | 1090 +++++++++++++++++ .../m3u/catchup_token.rs} | 0 .../network/m3u.rs => iptv/m3u/client.rs} | 0 backend/src/iptv/m3u/mod.rs | 9 + backend/src/iptv/mod.rs | 3 + .../{utils/network => iptv}/stalker/auth.rs | 43 +- .../network => iptv}/stalker/catalog.rs | 16 +- .../{utils/network => iptv}/stalker/client.rs | 26 +- .../network => iptv}/stalker/cmd_parser.rs | 2 +- .../network => iptv}/stalker/cookie_jar.rs | 2 +- .../{utils/network => iptv}/stalker/epg.rs | 14 +- .../{utils/network => iptv}/stalker/error.rs | 47 +- .../{utils/network => iptv}/stalker/mod.rs | 0 .../network => iptv}/stalker/playback.rs | 24 +- .../network => iptv}/stalker/presets.rs | 0 .../network => iptv}/stalker/profile.rs | 2 +- .../network => iptv}/stalker/recipes.rs | 18 +- .../network => iptv}/stalker/session.rs | 24 +- .../network => iptv}/stalker/url_factory.rs | 4 +- .../network/xtream.rs => iptv/xtream/mod.rs} | 1 + backend/src/modules.rs | 1 + backend/src/processing/parser/hls.rs | 197 ++- backend/src/processing/parser/m3u.rs | 45 + backend/src/processing/parser/stalker.rs | 2 +- backend/src/processing/parser/xtream.rs | 2 +- backend/src/processing/processor/playlist.rs | 85 +- backend/src/processing/processor/stalker.rs | 6 +- .../processing/processor/stalker_refresh.rs | 8 +- .../src/processing/processor/xtream_series.rs | 3 +- .../src/processing/processor/xtream_vod.rs | 3 +- backend/src/repository/library_repository.rs | 2 + .../src/repository/m3u_playlist_iterator.rs | 232 +++- backend/src/repository/playlist_repository.rs | 2 + backend/src/repository/stalker_repository.rs | 2 +- backend/src/repository/stream_history/mod.rs | 1 + .../repository/xtream_playlist_iterator.rs | 1 + backend/src/repository/xtream_repository.rs | 1 + backend/src/utils/db_viewer.rs | 1 + backend/src/utils/m3u_catchup.rs | 532 -------- backend/src/utils/mod.rs | 4 - backend/src/utils/network/mod.rs | 3 - backend/src/utils/network/request.rs | 49 +- docs/src/configuration/source.md | 37 +- docs/src/features.md | 1 + .../dashboard/stream_display/helpers.rs | 1 + frontend/src/hooks/use_server_status.rs | 2 + shared/src/model/playlist.rs | 43 +- shared/src/model/stream_info.rs | 39 + shared/src/model/stream_properties.rs | 38 + 73 files changed, 4346 insertions(+), 890 deletions(-) create mode 100644 backend/src/iptv/m3u/archive.rs create mode 100644 backend/src/iptv/m3u/catchup.rs rename backend/src/{utils/m3u_catchup_token.rs => iptv/m3u/catchup_token.rs} (100%) rename backend/src/{utils/network/m3u.rs => iptv/m3u/client.rs} (100%) create mode 100644 backend/src/iptv/m3u/mod.rs create mode 100644 backend/src/iptv/mod.rs rename backend/src/{utils/network => iptv}/stalker/auth.rs (91%) rename backend/src/{utils/network => iptv}/stalker/catalog.rs (98%) rename backend/src/{utils/network => iptv}/stalker/client.rs (97%) rename backend/src/{utils/network => iptv}/stalker/cmd_parser.rs (98%) rename backend/src/{utils/network => iptv}/stalker/cookie_jar.rs (99%) rename backend/src/{utils/network => iptv}/stalker/epg.rs (98%) rename backend/src/{utils/network => iptv}/stalker/error.rs (72%) rename backend/src/{utils/network => iptv}/stalker/mod.rs (100%) rename backend/src/{utils/network => iptv}/stalker/playback.rs (93%) rename backend/src/{utils/network => iptv}/stalker/presets.rs (100%) rename backend/src/{utils/network => iptv}/stalker/profile.rs (99%) rename backend/src/{utils/network => iptv}/stalker/recipes.rs (93%) rename backend/src/{utils/network => iptv}/stalker/session.rs (79%) rename backend/src/{utils/network => iptv}/stalker/url_factory.rs (97%) rename backend/src/{utils/network/xtream.rs => iptv/xtream/mod.rs} (99%) delete mode 100644 backend/src/utils/m3u_catchup.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index fd9e525a3..2b8d62bef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -270,6 +270,18 @@ - M3U catchup metadata is now stored under live stream properties and survives playlist rewrite/output generation. - XMLTV `catchup-id` is now imported, merged and exported alongside programme data. - Reverse-proxied M3U outputs can now expose local Tuliprox catchup/archive URLs instead of leaking provider archive URLs. + - Catchup URL templates now resolve indexed parameters and common player query forms such as `utc` / `lutc` and + `utcstart` with `offset` or `duration`. + - Native Flussonic archive playback supports HLS (`.m3u8`) and MPEG-TS (`.ts`) paths. This includes flat archive + requests used by TiviMate and nested Flussonic archive paths. + - Live M3U entries that provide `timeshift` without a separate catchup block now use that value as Flussonic-style + catchup metadata instead of being rejected as non-live archive requests. + - Archive timestamps are retained when Tuliprox follows same-origin child HLS playlists. Segment, key and + initialization URLs are left unchanged apart from normal relative URL resolution. + - Generated M3U playlists advertise the user's Tuliprox XMLTV endpoint through both `url-tvg` and `x-tvg-url` when + server information is available. + - Per-channel `#EXTVLCOPT:http-user-agent` values are imported and used for upstream HLS and MPEG-TS requests. The + directive is only written to playlists that still contain direct provider URLs. - **Per-User Output Clusters**: API proxy users can now be restricted to specific clusters on their assigned target via `output_clusters`. @@ -627,6 +639,9 @@ ## 🛠 Maintenance +- Moved provider-specific M3U, Xtream and Stalker protocol code from the generic utility namespace into dedicated IPTV + modules. + - **B+Tree v3 persistence engine**: - Consolidated the facade, v2 compatibility reader, v3 engine, migration, WAL, sorted index, and stress tests under `backend/src/repository/bplustree/`. diff --git a/backend/src/api/api_utils.rs b/backend/src/api/api_utils.rs index ea11344db..8cb3325ba 100644 --- a/backend/src/api/api_utils.rs +++ b/backend/src/api/api_utils.rs @@ -8,7 +8,7 @@ use crate::{ tee_stream, AppState, BoxedProviderStream, CustomVideoStreamType, PendingProviderReason, ProviderAllocation, ProviderConfig, ProviderHandle, ProviderStreamCustomReason, ProviderStreamFactoryOptions, ProviderStreamInfo, ProviderStreamState, SharedStreamManager, StreamDetails, - StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, + StreamError, StreamingStrategy, ThrottledStream, UserApiRequest, UserSession, MAX_HLS_MANIFEST_BYTES, }, }, auth::Fingerprint, @@ -22,7 +22,10 @@ use crate::{ MediaServerError, MediaServerErrorKind, MediaServerHttpClient, MediaServerImageRef, }, model::{AppConfig, ConfigInput, ConfigTarget, InputUserInfo, ProxyUserCredentials}, - processing::processor::re_resolve_stalker_url, + processing::{ + parser::hls::{rewrite_hls, RewriteHlsProps}, + processor::re_resolve_stalker_url, + }, utils::{ async_file_reader, async_file_writer, create_new_file_for_write, debug_if_enabled, get_file_extension, request, request::{content_type_from_ext, parse_range, send_with_retry_and_provider}, @@ -64,6 +67,7 @@ use std::{ net::SocketAddr, path::{Path, PathBuf}, sync::{Arc, LazyLock}, + time::Duration, }; use tokio::{ io::{AsyncReadExt, AsyncSeekExt}, @@ -1871,7 +1875,12 @@ fn should_defer_provider_open_for_grace_hold( return false; } - // v3.3.0 opened provider-affine VOD/Series/Catchup reopens immediately, even when + // Catch-up must open immediately so its payload can be classified before response headers are committed. + if item_type == PlaylistItemType::Catchup { + return false; + } + + // v3.3.0 opened provider-affine VOD/Series reopens immediately, even when // provider grace was temporarily in effect. Parking these requests in GracePending // was introduced later and breaks players like libmpv during seek/reopen retries. // Keep hold-stream behavior for live/admission paths, but restore direct-open behavior @@ -2050,6 +2059,7 @@ async fn create_stream_response_details( provider_name: Some(provider_name), request_url: None, session_headers: session_headers.cloned(), + provider_session_headers: HashMap::new(), grace_period: grace_period_options, provider_grace_active: false, disable_provider_grace: false, @@ -2081,24 +2091,26 @@ async fn create_stream_response_details( } else { false }; - let (stream, stream_info, reconnect_flag) = if defer_provider_stream_until_grace_check { + let (stream, stream_info, provider_session_headers, reconnect_flag) = + if defer_provider_stream_until_grace_check { debug_if_enabled!( "Deferring provider stream open until grace check completes for {}", sanitize_sensitive_info(resolve_request_url_for_logging(input, request_url.as_ref()).as_ref()) ); - (None, None, None) + (None, None, HashMap::new(), None) } else if is_media_server_stream_ref_url(request_url.as_ref()) { match open_media_server_stream_for_input(app_state, input, request_url.as_ref(), req_headers).await { - Ok((stream, stream_info)) => (Some(stream), stream_info, None), + Ok((stream, stream_info)) => (Some(stream), stream_info, HashMap::new(), None), Err(err) => { error!("Can't open media-server stream: {err}"); - (None, None, None) + (None, None, HashMap::new(), None) } } } else { let parsed_url = Url::parse(&request_url); let request_url_valid = parsed_url.is_ok(); - let ((mut stream, mut stream_info), mut reconnect_flag) = if let Ok(url) = parsed_url { + let ((mut stream, mut stream_info, mut provider_session_headers), mut reconnect_flag) = + if let Ok(url) = parsed_url { let default_user_agent = app_state.app_config.config.load().default_user_agent.clone(); let disabled_headers = app_state.get_disabled_headers(); let mut provider_stream_factory_options = @@ -2134,12 +2146,13 @@ async fn create_stream_response_details( ) .await { - None => (None, None), - Some((stream, info)) => (Some(stream), info), + None => (None, None, HashMap::new()), + Some(response) => + (Some(response.stream), response.info, response.provider_session_headers), }; (provider_stream, Some(reconnect_flag)) } else { - ((None, None), None) + ((None, None, HashMap::new()), None) }; let should_refresh_stalker = should_refresh_stalker_playback( input.input_type, @@ -2192,9 +2205,10 @@ async fn create_stream_response_details( options, ) .await; - if let Some((retry_stream, retry_info)) = retried { - stream = Some(retry_stream); - stream_info = retry_info; + if let Some(response) = retried { + stream = Some(response.stream); + stream_info = response.info; + provider_session_headers = response.provider_session_headers; reconnect_flag = Some(retry_reconnect_flag); request_url = refreshed_url; } else { @@ -2210,7 +2224,7 @@ async fn create_stream_response_details( } } } - (stream, stream_info, reconnect_flag) + (stream, stream_info, provider_session_headers, reconnect_flag) }; if log_enabled!(log::Level::Debug) { @@ -2241,6 +2255,7 @@ async fn create_stream_response_details( provider_name: guard_provider_name.clone(), request_url: Some(request_url.clone()), session_headers: session_headers.cloned(), + provider_session_headers, grace_period: grace_period_options, provider_grace_active, disable_provider_grace: false, @@ -2817,7 +2832,11 @@ pub(crate) async fn stream_response( input, &stream_channel, item_type, - crate::api::model::ProviderContentRepresentationMode::PreserveOrigin, + if item_type == PlaylistItemType::Catchup { + crate::api::model::ProviderContentRepresentationMode::Identity + } else { + crate::api::model::ProviderContentRepresentationMode::PreserveOrigin + }, share_stream, connection_permission, pinned_provider, @@ -2851,6 +2870,43 @@ pub(crate) async fn stream_response( } }; + if item_type == PlaylistItemType::Catchup { + if let Some(provider_stream) = stream_details.stream.take() { + let probe_deadline = Duration::from_millis(app_state.hls_proxy.origin_manifest_timeout_ms().max(1)); + match probe_catchup_payload(provider_stream, probe_deadline).await { + Ok(CatchupPayload::Direct(provider_stream)) => stream_details.stream = Some(provider_stream), + Ok(CatchupPayload::HlsManifest(manifest)) => { + return detected_catchup_hls_response(DetectedCatchupHlsResponseParams { + app_state, + stream_details, + manifest, + user, + target, + input, + fingerprint, + session_token, + virtual_id, + connection_permission, + connection_kind, + fallback_stream_url: stream_url, + }) + .await; + } + Err(err) => { + error!("Failed to inspect catch-up payload: {err}"); + cleanup_failed_detected_catchup_hls( + app_state, + &mut stream_details, + &user.username, + session_token, + ) + .await; + return StatusCode::BAD_GATEWAY.into_response(); + } + } + } + } + // When no provider stream is available, still create an ActiveClientStream if a grace period // needs to resolve (provider-grace with hold_stream, or user-grace). The grace task will // determine the correct mode (UserExhausted / ProviderExhausted / Inner) and serve the @@ -3068,6 +3124,188 @@ pub(crate) async fn stream_response( no_custom_video_fallback_status(&app_state.app_config).into_response() } +enum CatchupPayload { + Direct(BoxedProviderStream), + HlsManifest(Bytes), +} + +struct DetectedCatchupHlsResponseParams<'a> { + app_state: &'a Arc, + stream_details: StreamDetails, + manifest: Bytes, + user: &'a ProxyUserCredentials, + target: &'a ConfigTarget, + input: &'a ConfigInput, + fingerprint: &'a Fingerprint, + session_token: &'a str, + virtual_id: VirtualId, + connection_permission: UserConnectionPermission, + connection_kind: crate::api::model::ConnectionKind, + fallback_stream_url: &'a str, +} + +async fn detected_catchup_hls_response( + params: DetectedCatchupHlsResponseParams<'_>, +) -> axum::response::Response { + let DetectedCatchupHlsResponseParams { + app_state, + mut stream_details, + manifest, + user, + target, + input, + fingerprint, + session_token, + virtual_id, + connection_permission, + connection_kind, + fallback_stream_url, + } = params; + + let Some(provider) = stream_details.provider_name.clone() else { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + }; + let Some(server_info) = app_state.app_config.get_user_server_info(user) else { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + }; + let Ok(content) = std::str::from_utf8(&manifest) else { + cleanup_failed_detected_catchup_hls(app_state, &mut stream_details, &user.username, session_token).await; + return StatusCode::BAD_GATEWAY.into_response(); + }; + + let response_url = stream_details + .stream_info + .as_ref() + .and_then(|(_, _, response_url, _)| response_url.as_ref()) + .map_or_else(|| fallback_stream_url.to_string(), ToString::to_string); + let base_url = server_info.get_base_url(); + let encrypt_secret = app_state.get_encrypt_secret(); + let rewritten = rewrite_hls( + user, + &RewriteHlsProps { + secret: &encrypt_secret, + base_url: &base_url, + content, + hls_url: response_url, + target_id: target.id, + virtual_id, + input_id: input.id, + user_token: Some(session_token), + }, + ); + + let request_url = stream_details.request_url.as_deref().unwrap_or(fallback_stream_url); + let created_session_token = app_state + .active_users + .create_user_session(crate::api::model::CreateUserSessionParams { + user, + session_token, + virtual_id, + provider: &provider, + stream_url: request_url, + addr: &fingerprint.addr, + connection_permission, + connection_kind: Some(connection_kind), + socket_bound: false, + }) + .await; + if !stream_details.provider_session_headers.is_empty() { + app_state + .active_users + .update_session_provider_headers( + &user.username, + &created_session_token, + &stream_details.provider_session_headers, + ) + .await; + } + app_state + .active_provider + .refresh_provider_reservation( + &provider, + &created_session_token, + get_catchup_session_ttl_secs(app_state), + ) + .await; + app_state.connection_manager.release_provider_handle(stream_details.provider_handle.take()).await; + app_state + .active_users + .release_unbound_session_reservation( + &user.username, + &created_session_token, + None, + false, + ) + .await; + app_state + .active_users + .clear_unbound_session_addr(&user.username, &created_session_token, &fingerprint.addr) + .await; + + catchup_hls_manifest_response(rewritten) +} + +fn catchup_hls_manifest_response(content: String) -> axum::response::Response { + let mut response = try_unwrap_body!(axum::response::Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/vnd.apple.mpegurl") + .header(header::CACHE_CONTROL, "no-store, no-cache, must-revalidate") + .body(Body::from(content))); + mark_response_as_uncompressed(&mut response); + response +} + +async fn cleanup_failed_detected_catchup_hls( + app_state: &Arc, + stream_details: &mut StreamDetails, + username: &str, + session_token: &str, +) { + app_state.connection_manager.release_provider_handle(stream_details.provider_handle.take()).await; + app_state.active_users.terminate_session(username, session_token).await; + app_state.active_provider.clear_provider_reservation(session_token).await; +} + +async fn probe_catchup_payload( + stream: BoxedProviderStream, + deadline: Duration, +) -> Result { + tokio::time::timeout(deadline, probe_catchup_payload_inner(stream)) + .await + .map_err(|_| StreamError::Stream("catch-up payload probe timed out".to_string()))? +} + +async fn probe_catchup_payload_inner(mut stream: BoxedProviderStream) -> Result { + const HLS_SIGNATURE: &[u8] = b"#EXTM3U"; + + let mut prefix = BytesMut::new(); + while prefix.len() < HLS_SIGNATURE.len() { + let Some(chunk) = stream.next().await else { + return Ok(CatchupPayload::Direct(stream::once(async move { Ok(prefix.freeze()) }).chain(stream).boxed())); + }; + prefix.extend_from_slice(&chunk?); + } + + if !prefix.starts_with(HLS_SIGNATURE) { + return Ok(CatchupPayload::Direct(stream::once(async move { Ok(prefix.freeze()) }).chain(stream).boxed())); + } + if prefix.len() > MAX_HLS_MANIFEST_BYTES { + return Err(StreamError::Stream("catch-up HLS manifest exceeds size limit".to_string())); + } + + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if prefix.len().saturating_add(chunk.len()) > MAX_HLS_MANIFEST_BYTES { + return Err(StreamError::Stream("catch-up HLS manifest exceeds size limit".to_string())); + } + prefix.extend_from_slice(&chunk); + } + + Ok(CatchupPayload::HlsManifest(prefix.freeze())) +} + fn get_stream_throttle(app_state: &Arc) -> u64 { app_state .app_config @@ -3897,7 +4135,7 @@ pub fn redirect(url: &str) -> impl IntoResponse { .body(Body::empty())) } -pub async fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> bool { +pub fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> bool { // seek only for non-live streams if cluster == XtreamCluster::Live { return false; @@ -3914,6 +4152,14 @@ pub async fn is_seek_request(cluster: XtreamCluster, req_headers: &HeaderMap) -> false } +pub fn is_seekable_media_request( + cluster: XtreamCluster, + req_headers: &HeaderMap, + extension: Option<&str>, +) -> bool { + !extension.is_some_and(|ext| ext.eq_ignore_ascii_case(HLS_EXT)) && is_seek_request(cluster, req_headers) +} + pub fn bin_response(data: &T) -> impl IntoResponse + Send { match bin_serialize(data) { Ok(body) => ([(header::CONTENT_TYPE, CONTENT_TYPE_CBOR)], body).into_response(), @@ -4415,6 +4661,70 @@ mod tests { assert!(decoded.is_ok_and(|values| values.len() == 4_096)); } + #[tokio::test] + async fn catchup_payload_probe_detects_fragmented_hls() { + let source = stream::iter([ + Ok::<_, StreamError>(Bytes::from_static(b"#EX")), + Ok(Bytes::from_static(b"TM3U\nsegment.ts\n")), + ]) + .boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(matches!(&result, Ok(CatchupPayload::HlsManifest(_)))); + if let Ok(CatchupPayload::HlsManifest(manifest)) = result { + assert_eq!(manifest, b"#EXTM3U\nsegment.ts\n".as_slice()); + } + } + + #[tokio::test] + async fn catchup_payload_probe_replays_ts_bytes() { + let expected = Bytes::from_static(b"\x47direct-ts-payload"); + let source = stream::iter([Ok::<_, StreamError>(expected.clone())]).boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(matches!(&result, Ok(CatchupPayload::Direct(_)))); + if let Ok(CatchupPayload::Direct(mut stream)) = result { + let mut actual = Vec::new(); + while let Some(chunk) = stream.next().await { + if let Ok(chunk) = chunk { + actual.extend_from_slice(&chunk); + } + } + assert_eq!(actual, expected.as_ref()); + } + } + + #[tokio::test] + async fn catchup_payload_probe_replays_partial_signature_at_eof() { + let expected = Bytes::from_static(b"#EXT"); + let source = stream::iter([Ok::<_, StreamError>(expected.clone())]).boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(matches!(&result, Ok(CatchupPayload::Direct(_)))); + if let Ok(CatchupPayload::Direct(mut stream)) = result { + let actual = stream.next().await.and_then(Result::ok); + assert_eq!(actual.as_ref(), Some(&expected)); + assert!(stream.next().await.is_none()); + } + } + + #[tokio::test] + async fn catchup_payload_probe_rejects_oversized_manifest() { + let oversized = vec![b'x'; MAX_HLS_MANIFEST_BYTES]; + let source = stream::iter([ + Ok::<_, StreamError>(Bytes::from_static(b"#EXTM3U")), + Ok(Bytes::from(oversized)), + ]) + .boxed(); + + let result = probe_catchup_payload(source, std::time::Duration::from_secs(1)).await; + + assert!(result.is_err()); + } + #[tokio::test] async fn coalesced_stream_remains_finished_when_polled_again() { let stream = coalesce_byte_stream(stream::empty::>()); @@ -4460,28 +4770,45 @@ mod tests { )) } - #[tokio::test] - async fn test_is_seek_request() { + #[test] + fn test_is_seek_request() { let mut headers = HeaderMap::new(); // No range header - assert!(!is_seek_request(XtreamCluster::Video, &headers).await); + assert!(!is_seek_request(XtreamCluster::Video, &headers)); // Range: bytes=0- (Should be true now to allow session takeover on restart) headers.insert("range", "bytes=0-".parse().unwrap()); - assert!(is_seek_request(XtreamCluster::Video, &headers).await); + assert!(is_seek_request(XtreamCluster::Video, &headers)); // Range: bytes=100- (Should be true) headers.insert("range", "bytes=100-".parse().unwrap()); - assert!(is_seek_request(XtreamCluster::Video, &headers).await); + assert!(is_seek_request(XtreamCluster::Video, &headers)); // Range: bytes=100-200 (Should be true) headers.insert("range", "bytes=100-200".parse().unwrap()); - assert!(is_seek_request(XtreamCluster::Video, &headers).await); + assert!(is_seek_request(XtreamCluster::Video, &headers)); // Live cluster should always return false headers.insert("range", "bytes=100-".parse().unwrap()); - assert!(!is_seek_request(XtreamCluster::Live, &headers).await); + assert!(!is_seek_request(XtreamCluster::Live, &headers)); + } + + #[test] + fn hls_manifests_are_not_forced_as_seek_responses() { + let mut headers = HeaderMap::new(); + headers.insert("range", HeaderValue::from_static("bytes=0-")); + + assert!(!is_seekable_media_request( + XtreamCluster::Video, + &headers, + Some(HLS_EXT) + )); + assert!(is_seekable_media_request( + XtreamCluster::Video, + &headers, + Some(".ts") + )); } #[test] @@ -5664,6 +5991,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } @@ -5684,6 +6012,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } @@ -8241,6 +8570,7 @@ mod tests { provider_name: Some("provider_1".intern()), request_url: None, session_headers: None, + provider_session_headers: HashMap::new(), grace_period: GracePeriodOptions::default(), provider_grace_active: false, disable_provider_grace: false, @@ -8260,6 +8590,7 @@ mod tests { provider_name: Some("provider_1".intern()), request_url: None, session_headers: None, + provider_session_headers: HashMap::new(), grace_period: GracePeriodOptions::default(), provider_grace_active: false, disable_provider_grace: false, @@ -8286,6 +8617,7 @@ mod tests { provider_name: Some("provider_1".intern()), request_url: None, session_headers: None, + provider_session_headers: HashMap::new(), grace_period: GracePeriodOptions::default(), provider_grace_active: false, disable_provider_grace: false, @@ -9272,9 +9604,10 @@ mod tests { } #[test] - fn grace_hold_defers_live_but_not_provider_affine_session_reopens() { + fn grace_hold_defers_live_and_fresh_video_but_not_catchup_or_affine_reopens() { assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::LiveHls, false)); assert!(should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Video, false)); + assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Catchup, false)); assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Catchup, true)); assert!(!should_defer_provider_open_for_grace_hold(true, true, PlaylistItemType::Video, true)); assert!(!should_defer_provider_open_for_grace_hold(true, false, PlaylistItemType::Video, true)); @@ -9349,6 +9682,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "1".intern(), + upstream_user_agent: None, }; let hls_ext = shared::defaults::HLS_EXT.to_string(); diff --git a/backend/src/api/endpoints/api_playlist_utils.rs b/backend/src/api/endpoints/api_playlist_utils.rs index c25563a99..4d6ae724a 100644 --- a/backend/src/api/endpoints/api_playlist_utils.rs +++ b/backend/src/api/endpoints/api_playlist_utils.rs @@ -1,13 +1,13 @@ use crate::api::model::AppState; use crate::{ api::api_utils::{empty_json_list_response, stream_json_or_bin_response_stream, stream_json_or_bin_response_try_stream}, + iptv::{m3u, xtream}, model::{ConfigInput, ConfigTarget}, processing::processor::{download_stalker_playlist, StalkerCluster}, repository::{ iter_raw_m3u_input_playlist, iter_raw_m3u_target_playlist, iter_raw_xtream_input_playlist, iter_raw_xtream_target_playlist, }, - utils::{m3u, xtream}, }; use axum::response::IntoResponse; use serde_json::json; diff --git a/backend/src/api/endpoints/hls_api.rs b/backend/src/api/endpoints/hls_api.rs index edbde0951..9b65185ce 100644 --- a/backend/src/api/endpoints/hls_api.rs +++ b/backend/src/api/endpoints/hls_api.rs @@ -8,9 +8,11 @@ use super::xtream_url::{get_query_path, get_xtream_player_api_stream_url, ApiStr use crate::{ api::{ api_utils::{ - connection_priority_for_kind, create_api_proxy_user, create_playback_session_fingerprint, - create_session_fingerprint, force_provider_stream_response, get_headers_from_request, - get_hls_session_ttl_secs, get_stream_alternative_url, is_hls_stream_share_enabled, is_seek_request, + connection_priority_for_kind, create_api_proxy_user, create_m3u_catchup_session_key, + create_playback_session_fingerprint, create_session_fingerprint, force_provider_stream_response, + get_headers_from_request, + get_hls_session_ttl_secs, get_stream_alternative_url, is_hls_stream_share_enabled, + is_seekable_media_request, local_stream_response, record_connect_failed_attempt, resolve_playback_request_admission, try_option_bad_request, try_unwrap_body, ConnectFailedAttempt, EvictionReentryGuard, HeaderFilter, }, @@ -86,8 +88,8 @@ use shared::{ TargetType, UserConnectionPermission, XtreamCluster, }, utils::{ - generate_random_string, is_hls_url, replace_url_extension, sanitize_sensitive_info, Internable, - PROVIDER_SCHEME_PREFIX, + extract_extension_from_url, generate_random_string, is_hls_url, replace_url_extension, + sanitize_sensitive_info, Internable, PROVIDER_SCHEME_PREFIX, }, }; use std::{borrow::Cow, collections::HashMap, sync::Arc, time::Duration}; @@ -101,10 +103,28 @@ fn is_m3u_catchup_session_token(session_token: &str) -> bool { session_token.starts_with("m3u-catchup|") || session_token.starts_with("catchup|") } -fn query_flag_is_archive(key: &str) -> bool { key.eq_ignore_ascii_case("utc") } +fn legacy_hls_route_allowed_with_cache( + cache_enabled: bool, + decoded_session_token: Option<&str>, + existing_session_token: Option<&str>, +) -> bool { + !cache_enabled + || decoded_session_token.is_some_and(|decoded| { + existing_session_token.is_some_and(|existing| { + decoded == existing && is_m3u_catchup_session_token(existing) + }) + }) +} + +fn query_flag_is_archive(key: &str) -> bool { + key.eq_ignore_ascii_case("utc") || key.eq_ignore_ascii_case("utcstart") +} fn query_flag_marks_start_context(key: &str) -> bool { - key.eq_ignore_ascii_case("end") || key.eq_ignore_ascii_case("duration") || key.eq_ignore_ascii_case("lutc") + key.eq_ignore_ascii_case("end") + || key.eq_ignore_ascii_case("duration") + || key.eq_ignore_ascii_case("lutc") + || key.eq_ignore_ascii_case("offset") } pub(in crate::api) fn m3u_archive_epg_reference_ts(stream_url: &str) -> Option { @@ -129,7 +149,7 @@ pub(in crate::api) fn m3u_archive_epg_reference_ts(stream_url: &str) -> Option() { return Some(ts); } - } else if key.eq_ignore_ascii_case("start") { + } else if key.eq_ignore_ascii_case("start") || key.eq_ignore_ascii_case("timestamp") { start_ts = value.parse::().ok(); } else if query_flag_marks_start_context(&key) { has_start_context = true; @@ -449,8 +469,20 @@ async fn prepare_hls_resource_access( Ok(HlsResourceAccess { session, access_context }) } -fn create_hls_cache_user_session_token(fingerprint: &Fingerprint, username: &str, virtual_id: u32) -> String { - let base = create_playback_session_fingerprint(fingerprint, username, virtual_id, PlaylistItemType::LiveHls, None); +fn create_hls_cache_user_session_token( + fingerprint: &Fingerprint, + username: &str, + virtual_id: u32, + existing_session_token: Option<&str>, + archive_reference: Option, +) -> String { + let base = if let Some(token) = existing_session_token.filter(|token| is_m3u_catchup_session_token(token)) { + token.to_string() + } else if let Some(timestamp) = archive_reference { + create_m3u_catchup_session_key(fingerprint, username, virtual_id, &format!("archive|{timestamp}|0")) + } else { + create_playback_session_fingerprint(fingerprint, username, virtual_id, PlaylistItemType::LiveHls, None) + }; format!("{base}|hls-cache|{}", generate_random_string(16)) } @@ -1316,17 +1348,30 @@ async fn build_hls_cache_stream_channel( origin_source: &HlsOriginSource, proxy_session_id: &ProxySessionId, ) -> StreamChannel { - if let Some((_, target)) = app_state.app_config.get_target_for_username(&access.username) { + let mut channel = if let Some((_, target)) = app_state.app_config.get_target_for_username(&access.username) { if let Some(mut channel) = get_stream_channel(app_state, &target, access.virtual_id).await { channel.url = Arc::from(hls_cache_stream_stats_url(proxy_session_id)); - channel.item_type = PlaylistItemType::LiveHls; - channel.cluster = XtreamCluster::try_from(PlaylistItemType::LiveHls).unwrap_or(channel.cluster); - return channel; + channel + } else { + fallback_hls_cache_stream_channel(target.id, access.virtual_id, origin_source, proxy_session_id) } - return fallback_hls_cache_stream_channel(target.id, access.virtual_id, origin_source, proxy_session_id); - } + } else { + fallback_hls_cache_stream_channel(0, access.virtual_id, origin_source, proxy_session_id) + }; - fallback_hls_cache_stream_channel(0, access.virtual_id, origin_source, proxy_session_id) + if access.epg_reference_ts.is_some() + || access.archive_origin_url.is_some() + || is_m3u_catchup_session_token(&access.user_session_token) + { + channel.item_type = PlaylistItemType::Catchup; + channel.cluster = XtreamCluster::Video; + channel.epg_reference_ts = access.epg_reference_ts; + } else { + channel.item_type = PlaylistItemType::LiveHls; + channel.cluster = XtreamCluster::try_from(PlaylistItemType::LiveHls).unwrap_or(channel.cluster); + channel.epg_reference_ts = None; + } + channel } fn fallback_hls_cache_stream_channel( @@ -1352,6 +1397,7 @@ fn fallback_hls_cache_stream_channel( technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } @@ -2013,6 +2059,7 @@ fn build_hls_manifest_request_headers( req_headers: &HeaderMap, disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, default_user_agent: Option<&str>, + upstream_user_agent: Option<&str>, ) -> HeaderMap { let input_headers = input_headers .iter() @@ -2027,6 +2074,7 @@ fn build_hls_manifest_request_headers( let forwarded = get_headers_from_request(req_headers, &filter_header); let mut headers = request::get_request_headers(Some(&input_headers), Some(&forwarded), disabled_headers, default_user_agent); + request::overlay_upstream_user_agent(&mut headers, upstream_user_agent, disabled_headers); scrub_hls_origin_headers(&mut headers, disabled_headers); force_identity_without_range(&mut headers); headers @@ -2145,11 +2193,26 @@ fn build_hls_origin_source(input: &ConfigInput, stream_ref: impl Into) - HlsOriginSource::new(input.id, Arc::clone(&input.name), stream_ref, hls_origin_source_kind(input.input_type)) } +fn build_hls_origin_source_for_playback( + input: &ConfigInput, + stream_ref: impl Into, + archive_reference: Option, + archive_url: Option<&str>, +) -> HlsOriginSource { + let source = build_hls_origin_source(input, stream_ref); + match (archive_reference, archive_url) { + (Some(timestamp), Some(url)) => source.with_archive_request(timestamp, url), + (Some(timestamp), None) => source.with_archive_reference(timestamp), + (None, _) => source, + } +} + /// Keeps target routing identity separate from the immutable input content identity. #[derive(Debug, Clone, Eq, PartialEq)] pub(in crate::api) struct HlsEntryStreamIdentity { virtual_id: u32, input_stream_id: Arc, + upstream_user_agent: Option>, } impl HlsEntryStreamIdentity { @@ -2158,16 +2221,20 @@ impl HlsEntryStreamIdentity { if input_stream_id.trim().is_empty() { return None; } - Some(Self { virtual_id, input_stream_id }) + Some(Self { virtual_id, input_stream_id, upstream_user_agent: None }) } pub(in crate::api) fn from_playlist_item(item: &impl PlaylistEntry) -> Option { - Self::new(item.get_virtual_id(), item.get_input_stream_id()?) + let mut identity = Self::new(item.get_virtual_id(), item.get_input_stream_id()?)?; + identity.upstream_user_agent = item.get_upstream_user_agent().map(Internable::intern); + Some(identity) } pub(in crate::api) const fn virtual_id(&self) -> u32 { self.virtual_id } fn stream_ref(&self) -> &str { self.input_stream_id.as_ref() } + + fn upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } } /// Resolves the configured input together with both identities of one target entry. @@ -3648,7 +3715,7 @@ async fn try_hls_cache_entry_redirect( user: &ProxyUserCredentials, origin_source: HlsOriginSource, virtual_id: u32, - _existing_user_session: Option<&UserSession>, + existing_user_session: Option<&UserSession>, request_url: &str, input: &ConfigInput, connection_permission: UserConnectionPermission, @@ -3665,7 +3732,13 @@ async fn try_hls_cache_entry_redirect( let now_ms = current_time_millis(); let origin_connection_kind = hls_entry_origin_connection_kind(connection_permission, connection_kind); let access_lease_id = new_hls_access_lease_id(); - let session_token = create_hls_cache_user_session_token(fingerprint, &user.username, virtual_id); + let session_token = create_hls_cache_user_session_token( + fingerprint, + &user.username, + virtual_id, + existing_user_session.map(|session| session.token.as_str()), + origin_source.archive_reference, + ); let session_token = prepare_hls_cache_user_session( app_state, fingerprint, @@ -3689,6 +3762,10 @@ async fn try_hls_cache_entry_redirect( virtual_id, now_ms, hls_pending_bootstrap_window_ms(), + ) + .with_archive_playback( + origin_source.archive_reference, + origin_source.archive_reference.map(|_| request_url.to_string()), ); if let Some(connection_kind) = origin_connection_kind { lease = lease.with_origin_acquire_policy(connection_kind, connection_priority_for_kind(user, connection_kind)); @@ -5168,7 +5245,7 @@ pub(in crate::api) async fn handle_hls_stream_request( target: &ConfigTarget, user_session: Option<&UserSession>, hls_url: &str, - _archive_reference: Option, + archive_reference: Option, stream_identity: HlsEntryStreamIdentity, input: &ConfigInput, req_headers: &HeaderMap, @@ -5192,7 +5269,9 @@ pub(in crate::api) async fn handle_hls_stream_request( } let url = ensure_hls_manifest_extension(&normalized_hls_url); let hls_cache_origin = build_hls_origin_resolution(input, &url); - let hls_origin_source = hls_cache_origin.as_ref().map(|_| build_hls_origin_source(input, stream_ref.clone())); + let hls_origin_source = hls_cache_origin + .as_ref() + .map(|_| build_hls_origin_source_for_playback(input, stream_ref.clone(), archive_reference, Some(url.as_str()))); let server_info = app_state.app_config.get_user_server_info(user); let Some(server_info) = server_info else { return axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response(); @@ -5205,6 +5284,7 @@ pub(in crate::api) async fn handle_hls_stream_request( req_headers, disabled_headers.as_ref(), default_user_agent.as_deref(), + stream_identity.upstream_user_agent(), ); if hls_cache_enabled_for_target(app_state, target) { @@ -5218,7 +5298,11 @@ pub(in crate::api) async fn handle_hls_stream_request( origin_source, virtual_id, user_session, - hls_cache_origin.as_ref().map_or(url.as_str(), |origin| origin.session_entry_url.as_str()), + if archive_reference.is_some() { + url.as_str() + } else { + hls_cache_origin.as_ref().map_or(url.as_str(), |origin| origin.session_entry_url.as_str()) + }, input, connection_permission, connection_kind, @@ -5510,6 +5594,7 @@ async fn resolve_stream_channel( technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }, }; @@ -5554,13 +5639,21 @@ async fn resolve_hls_playback_manifest_request_context( let Some(channel) = get_stream_channel(app_state, &target, access_context.virtual_id).await else { return Err(StatusCode::NOT_FOUND); }; - let origin_playlist_url = + let origin_playlist_url = if let Some(archive_url) = access_context.archive_origin_url.as_ref() { + archive_url.clone() + } else { resolve_hls_origin_playlist_url(app_state, &target, &input, access_context.virtual_id, channel.url.as_ref()) - .await?; + .await? + }; let Some(hls_cache_origin) = build_hls_origin_resolution(&input, &origin_playlist_url) else { return Err(StatusCode::SERVICE_UNAVAILABLE); }; - let origin_source = build_hls_origin_source(&input, access_context.stream_ref.clone()); + let origin_source = build_hls_origin_source_for_playback( + &input, + access_context.stream_ref.clone(), + access_context.epg_reference_ts, + Some(&origin_playlist_url), + ); let Some(server_info) = app_state.app_config.get_user_server_info(&user) else { return Err(StatusCode::SERVICE_UNAVAILABLE); }; @@ -5571,6 +5664,7 @@ async fn resolve_hls_playback_manifest_request_context( req_headers, disabled_headers.as_ref(), default_user_agent.as_deref(), + channel.upstream_user_agent.as_deref(), ); let original_hls_entry_path = build_virtual_hls_entry_path(&target, &input, &user, access_context.virtual_id); @@ -5725,7 +5819,17 @@ async fn hls_api_stream_resolved( let Some(decoded_hls_token) = get_hls_session_token_and_url_from_token(&encrypt_secret, &token) else { return axum::http::StatusCode::BAD_REQUEST.into_response(); }; - if hls_cache_enabled_for_target(&app_state, &target) { + let lookup_session_token = decoded_hls_token + .0 + .clone() + .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, virtual_id, false)); + let mut user_session = + app_state.active_users.get_and_update_user_session(&user.username, &lookup_session_token).await; + if !legacy_hls_route_allowed_with_cache( + hls_cache_enabled_for_target(&app_state, &target), + decoded_hls_token.0.as_deref(), + user_session.as_ref().map(|session| session.token.as_str()), + ) { return hls_custom_video_manifest_redirect_response_for_username( &app_state, &user.username, @@ -5733,12 +5837,6 @@ async fn hls_api_stream_resolved( StatusCode::NOT_FOUND, ); } - let lookup_session_token = decoded_hls_token - .0 - .clone() - .unwrap_or_else(|| create_session_fingerprint(&fingerprint, &user.username, virtual_id, false)); - let mut user_session = - app_state.active_users.get_and_update_user_session(&user.username, &lookup_session_token).await; if let Some(session) = &mut user_session { let decoded_archive_reference = m3u_archive_epg_reference_ts(&decoded_hls_token.1); @@ -5796,7 +5894,12 @@ async fn hls_api_stream_resolved( app_state.connection_manager.touch_http_activity(&user.username, &session.token, &fingerprint.addr).await; let stream_channel = resolve_stream_channel(&app_state, &target, &input, virtual_id, &hls_url, archive_reference).await; - if is_seek_request(stream_channel.cluster, &req_headers).await { + if is_seekable_media_request( + stream_channel.cluster, + &req_headers, + extract_extension_from_url(&hls_url), + ) + { // partial request means we are in reverse proxy mode, seek happened return force_provider_stream_response( &fingerprint, @@ -6053,6 +6156,55 @@ mod tests { assert_eq!(m3u_archive_epg_reference_ts("http://provider/live/42.m3u8?start=1700000000"), None); } + #[test] + fn archive_epg_reference_supports_contextual_start_aliases() { + assert_eq!( + m3u_archive_epg_reference_ts( + "http://provider/live/42.m3u8?offset=-3600&utcstart=1717200000" + ), + Some(1_717_200_000) + ); + assert_eq!( + m3u_archive_epg_reference_ts( + "http://provider/live/42.m3u8?timestamp=1717200000&offset=120" + ), + Some(1_717_200_000) + ); + } + + #[test] + fn hls_cache_session_tokens_separate_live_and_archive_playback() { + let fingerprint = test_fingerprint(); + let live = super::create_hls_cache_user_session_token(&fingerprint, "user", 31, None, None); + let archive = super::create_hls_cache_user_session_token( + &fingerprint, + "user", + 31, + None, + Some(1_784_898_000), + ); + + assert!(!super::is_m3u_catchup_session_token(&live)); + assert!(super::is_m3u_catchup_session_token(&archive)); + assert_ne!(live, archive); + } + + #[test] + fn hls_cache_session_token_preserves_existing_m3u_catchup_identity() { + let fingerprint = test_fingerprint(); + let existing = "m3u-catchup|fp|user|31|archive|1784898000|3600"; + let token = super::create_hls_cache_user_session_token( + &fingerprint, + "user", + 31, + Some(existing), + Some(1_784_898_000), + ); + + assert!(token.starts_with(existing)); + assert!(token.contains("|hls-cache|")); + } + #[test] fn extract_hls_provider_session_headers_converts_set_cookie_to_cookie_header() { let mut headers = HeaderMap::new(); @@ -6422,10 +6574,15 @@ mod tests { cloudflare_header: true, custom_header: vec!["X-Origin-Secret".to_string()], }; - let headers = - build_hls_manifest_request_headers(&input_headers, &request_headers, Some(&disabled), Some("Default-UA")); + let headers = build_hls_manifest_request_headers( + &input_headers, + &request_headers, + Some(&disabled), + Some("Default-UA"), + Some("Channel-UA"), + ); - assert_eq!(headers.get(header::USER_AGENT).expect("user agent"), "Input-UA"); + assert_eq!(headers.get(header::USER_AGENT).expect("user agent"), "Channel-UA"); assert_eq!(headers.get("accept-language").expect("accept language"), "de"); assert_eq!(headers.get(header::ACCEPT_ENCODING).expect("accept encoding"), "identity"); assert!(!headers.contains_key(header::RANGE)); @@ -8884,9 +9041,82 @@ mod tests { virtual_id: 12345, lease_id: access_lease_id, family_key: HlsPlaybackFamilyKey::new("hls-user", client_fingerprint), + epg_reference_ts: None, + archive_origin_url: None, } } + #[tokio::test] + async fn hls_cache_stream_channel_uses_archive_epg_context() { + let app_state = test_app_state(); + let mut access = test_hls_access_context( + ProxySessionId("proxy-archive".to_string()), + HlsAccessLeaseId("lease-archive".to_string()), + ); + access.epg_reference_ts = Some(1_784_898_000); + access.archive_origin_url = Some( + "http://provider/channel/timeshift_abs-1784898000.m3u8".to_string(), + ); + let origin_source = HlsOriginSource::new( + 1, + Arc::from("test-input"), + "80510", + HlsOriginSourceKind::M3uMediaPlaylist, + ) + .with_archive_reference(1_784_898_000); + + let channel = super::build_hls_cache_stream_channel( + &app_state, + &access, + &origin_source, + &ProxySessionId("proxy-archive".to_string()), + ) + .await; + + assert_eq!(channel.item_type, PlaylistItemType::Catchup); + assert_eq!(channel.cluster, XtreamCluster::Video); + assert_eq!(channel.epg_reference_ts, Some(1_784_898_000)); + } + + #[tokio::test] + async fn hls_cache_manifest_context_restores_leased_archive_origin() -> Result<(), StatusCode> { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + let input = ConfigInput { + id: 1, + name: Arc::from("test-input"), + input_type: InputType::M3u, + enabled: true, + ..ConfigInput::default() + }; + let mut target = test_m3u_hls_share_target(); + target.name = "default".to_string(); + store_test_sources_with_target(&app_state, input.clone(), target.clone()); + cache_test_m3u_hls_item( + &app_state, + &target, + test_m3u_hls_item(&input, 12345, "80510", "http://provider/channel/mono.m3u8"), + ) + .await; + + let archive_url = "http://provider/channel/timeshift_abs-1784898000.m3u8"; + let mut access = test_hls_access_context( + ProxySessionId("proxy-archive".to_string()), + HlsAccessLeaseId("lease-archive".to_string()), + ); + access.stream_ref = "80510".to_string(); + access.epg_reference_ts = Some(1_784_898_000); + access.archive_origin_url = Some(archive_url.to_string()); + + let context = + super::resolve_hls_playback_manifest_request_context(&app_state, &access, &HeaderMap::new()).await?; + + assert_eq!(context.hls_url, archive_url); + assert_eq!(context.origin_source.stream_ref, "80510"); + assert_eq!(context.origin_source.archive_reference, Some(1_784_898_000)); + Ok(()) + } + async fn prepare_pending_test_hls_access_lease( app_state: &Arc, proxy_session_id: &ProxySessionId, @@ -9457,6 +9687,73 @@ mod tests { assert!(app_state.active_users.active_streams().await.is_empty()); } + #[tokio::test] + async fn hls_cache_archive_entry_uses_distinct_identity_and_preserves_origin() -> Result<(), &'static str> { + let app_state = test_app_state(); + enable_hls_cache(&app_state); + configure_default_test_server(&app_state); + let mut user = ProxyUserCredentials::default(); + user.username = "hls-user".to_string(); + user.password = "hls-pass".to_string(); + let input = test_hls_input(); + let target = test_hls_share_target(true); + let archive_url = "http://origin.example.com/live/user/pass/timeshift_abs-1784898000.m3u8"; + let stream_identity = super::HlsEntryStreamIdentity::new(12345, "80510") + .ok_or("valid archive input stream identity")?; + + let response = super::handle_hls_stream_request( + &test_fingerprint(), + &app_state, + &user, + &target, + None, + archive_url, + Some(1_784_898_000), + stream_identity, + &input, + &HeaderMap::new(), + UserConnectionPermission::Allowed, + Some(ConnectionKind::Normal), + &super::build_virtual_hls_entry_path(&target, &input, &user, 12345), + ) + .await + .into_response(); + + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + let location = response + .headers() + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or("archive redirect location")?; + let proxy_session_id = ProxySessionId(proxy_session_id_from_redirect_location(location).to_string()); + let live_source = super::build_hls_origin_source(&input, "80510"); + let archive_source = super::build_hls_origin_source_for_playback( + &input, + "80510", + Some(1_784_898_000), + Some(archive_url), + ); + assert_ne!(proxy_session_id, build_proxy_session_id(&live_source.session_key(), &app_state.get_encrypt_secret())); + assert_eq!( + proxy_session_id, + build_proxy_session_id(&archive_source.session_key(), &app_state.get_encrypt_secret()) + ); + + let access_lease_id = HlsAccessLeaseId(access_lease_id_from_redirect_location(location).to_string()); + let lease = app_state + .hls_proxy + .access_leases() + .write() + .await + .response_snapshot(&access_lease_id, &proxy_session_id, super::current_time_millis()) + .ok_or("archive access lease")?; + assert_eq!(lease.stream_ref, "80510"); + assert_eq!(lease.epg_reference_ts, Some(1_784_898_000)); + assert_eq!(lease.archive_origin_url.as_deref(), Some(archive_url)); + assert!(super::is_m3u_catchup_session_token(&lease.user_session_token)); + Ok(()) + } + #[tokio::test] async fn hls_cache_entry_denies_access_lease_when_grace_has_no_origin_slot() { let app_state = test_app_state(); @@ -9791,6 +10088,28 @@ mod tests { assert!(app_state.active_users.active_streams().await.is_empty()); } + #[test] + fn cache_enabled_legacy_hls_route_only_allows_existing_m3u_catchup_session() { + assert!(super::legacy_hls_route_allowed_with_cache( + true, + Some("m3u-catchup|session"), + Some("m3u-catchup|session") + )); + assert!(super::legacy_hls_route_allowed_with_cache( + true, + Some("catchup|session"), + Some("catchup|session") + )); + assert!(!super::legacy_hls_route_allowed_with_cache( + true, + Some("m3u-catchup|session"), + Some("m3u-catchup|other") + )); + assert!(!super::legacy_hls_route_allowed_with_cache(true, Some("legacy-session"), Some("legacy-session"))); + assert!(!super::legacy_hls_route_allowed_with_cache(true, None, None)); + assert!(super::legacy_hls_route_allowed_with_cache(false, None, None)); + } + #[tokio::test] async fn hls_cache_canonical_manifest_rejects_when_target_hls_share_disabled() { let app_state = test_app_state(); @@ -9807,6 +10126,8 @@ mod tests { virtual_id: 12345, lease_id: HlsAccessLeaseId("access-lease".to_string()), family_key: HlsPlaybackFamilyKey::new("hls-user", test_fingerprint().key), + epg_reference_ts: None, + archive_origin_url: None, }; let Err(err) = diff --git a/backend/src/api/endpoints/m3u_api.rs b/backend/src/api/endpoints/m3u_api.rs index b09c5da94..020c0ed33 100644 --- a/backend/src/api/endpoints/m3u_api.rs +++ b/backend/src/api/endpoints/m3u_api.rs @@ -4,7 +4,7 @@ use crate::{ admission_failure_response, create_m3u_catchup_session_key, coalesce_byte_stream, create_playback_session_fingerprint, create_session_fingerprint, force_provider_stream_response, get_session_reservation_ttl_secs, - get_user_target, get_user_target_by_credentials, is_seek_request, is_session_based_playback, + get_user_target, get_user_target_by_credentials, is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, local_stream_response, redirect, redirect_response, resource_response, resolve_initial_stalker_playback_url, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, @@ -20,9 +20,16 @@ use crate::{ model::{AppState, UserApiRequest, UserApiRequestQueryOrBody}, }, auth::{check_network_access_only, resolve_api_user_context, ApiUserAuthError, Fingerprint}, + iptv::m3u::{ + build_provider_flussonic_archive_url, decode_m3u_catchup_token, has_m3u_catchup_marker, + is_flussonic_live_file, parse_flat_flussonic_archive, parse_flussonic_archive_file, + resolve_m3u_catchup_url, FlussonicArchiveKind, M3uCatchupToken, + }, model::{ConfigTarget, ProxyUserCredentials}, repository::{m3u_get_item_for_stream_id, m3u_load_rewrite_playlist, storage_const}, - utils::{debug_if_enabled, decode_m3u_catchup_token, has_m3u_catchup_marker, resolve_m3u_catchup_url, M3uCatchupToken, PROVIDER_SCHEME_PREFIX}, + utils::{ + debug_if_enabled, PROVIDER_SCHEME_PREFIX, + }, }; use axum::response::IntoResponse; use bytes::Bytes; @@ -31,8 +38,8 @@ use log::{debug, error}; use shared::error::TuliproxError; use shared::{ model::{ - ConnectFailureReason, FieldGetAccessor, PlaylistEntry, PlaylistItemType, TargetType, UserConnectionPermission, - XtreamCluster, + CatchupProperties, ConnectFailureReason, FieldGetAccessor, PlaylistEntry, PlaylistItemType, StreamProperties, + TargetType, UserConnectionPermission, XtreamCluster, }, utils::{concat_path, extract_extension_from_url, sanitize_sensitive_info}, defaults::{HLS_EXT} @@ -123,6 +130,19 @@ async fn m3u_api_post( m3u_api(user, target, &app_state, &api_req.content_type).await.into_response() } +fn effective_playback_extension<'a>( + item_type: PlaylistItemType, + stream_url: &'a str, + requested_extension: Option<&'a str>, +) -> Option<&'a str> { + let upstream_extension = extract_extension_from_url(stream_url); + if item_type == PlaylistItemType::Catchup { + upstream_extension.or(requested_extension) + } else { + requested_extension.or(upstream_extension) + } +} + #[allow(clippy::too_many_arguments, clippy::too_many_lines)] pub(in crate::api) async fn m3u_api_stream_loaded( user: Arc, @@ -142,7 +162,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( } let is_hls_manifest_request = - stream_ext == Some(HLS_EXT) || (stream_ext.is_none() && extract_extension_from_url(&pli.url) == Some(HLS_EXT)); + effective_playback_extension(pli.item_type, &pli.url, stream_ext) == Some(HLS_EXT); if !user.allows_item_type(pli.item_type) { if is_hls_manifest_request { @@ -246,8 +266,8 @@ pub(in crate::api) async fn m3u_api_stream_loaded( "M3U playback for virtual_id={virtual_id}, item_type={}", pli.item_type ); - let extracted_ext = extract_extension_from_url(&pli.url).unwrap_or_default(); - let extension = stream_ext.unwrap_or(extracted_ext); + let effective_stream_ext = effective_playback_extension(pli.item_type, &pli.url, stream_ext); + let extension = effective_stream_ext.unwrap_or_default(); let session_key = if pli.item_type == PlaylistItemType::Catchup { create_m3u_catchup_session_key( fingerprint, @@ -313,7 +333,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( ConnectFailureReason::ProviderConnectionsExhausted, ); } - if session.virtual_id == virtual_id && is_seek_request(cluster, req_headers).await { + if session.virtual_id == virtual_id && is_seekable_media_request(cluster, req_headers, Some(extension)) { // partial request means we are in reverse proxy mode, seek happened return force_provider_stream_response( fingerprint, @@ -399,7 +419,7 @@ pub(in crate::api) async fn m3u_api_stream_loaded( target: &target, input: &input, user: &user, - stream_ext, + stream_ext: effective_stream_ext, req_context: context, action_path: "", // TODO is there timeshift or something like that ? }; @@ -479,19 +499,76 @@ fn resolve_effective_source_url<'a>( input.resolve_url(&item.url) } +fn resolve_m3u_catchup_properties<'a>( + item_type: PlaylistItemType, + time_shift: &'a Arc, + additional_properties: Option<&'a StreamProperties>, +) -> Option> { + if let Some(StreamProperties::Live(live)) = additional_properties { + if let Some(catchup) = live.catchup.as_ref() { + return Some(Cow::Borrowed(catchup)); + } + } + + (item_type.is_live() && !time_shift.trim().is_empty()).then(|| { + Cow::Owned(CatchupProperties { + mode: Some(Arc::from("fs")), + days: Some(Arc::clone(time_shift)), + ..CatchupProperties::default() + }) + }) +} + +fn playlist_item_native_flussonic_mode(item: &shared::model::M3uPlaylistItem) -> Option<&'static str> { + match item.additional_properties.as_ref()? { + StreamProperties::Live(live) => live.catchup.as_ref()?.native_flussonic_player_mode(), + _ => None, + } +} + +fn native_live_file_matches_mode(mode: &str, file: &str) -> bool { + let supported = if mode == "flussonic-ts" { + &["mpegts", "index.ts", "video.ts", "mono.ts"][..] + } else { + &["index.m3u8", "video.m3u8", "mono.m3u8"] + }; + supported.iter().any(|supported_file| file.trim().eq_ignore_ascii_case(supported_file)) +} + +fn native_archive_matches_mode(mode: &str, archive: &FlussonicArchiveKind) -> bool { + let expected_extension = if mode == "flussonic-ts" { ".ts" } else { HLS_EXT }; + matches!(mode, "flussonic" | "flussonic-ts") && archive.extension() == expected_extension +} + +fn apply_native_flussonic_archive( + item: &shared::model::M3uPlaylistItem, + input: &crate::model::ConfigInput, + archive: &FlussonicArchiveKind, +) -> Result<(shared::model::M3uPlaylistItem, String), TuliproxError> { + let source_url = resolve_effective_source_url(item, input)?; + let archive_url = build_provider_flussonic_archive_url(source_url.as_ref(), archive).ok_or_else(|| { + TuliproxError::RepositoryM3u("Failed to build native Flussonic archive URL".to_string()) + })?; + let mut catchup_item = item.clone(); + catchup_item.item_type = PlaylistItemType::Catchup; + catchup_item.url = archive_url.into(); + Ok((catchup_item, archive.discriminator())) +} + fn resolve_loaded_m3u_catchup( pli: &shared::model::M3uPlaylistItem, input: &crate::model::ConfigInput, raw_query: Option<&str>, ) -> Result<(shared::model::M3uPlaylistItem, String), TuliproxError> { - let Some(shared::model::StreamProperties::Live(live)) = pli.additional_properties.as_ref() else { + let Some(catchup) = resolve_m3u_catchup_properties(pli.item_type, &pli.time_shift, pli.additional_properties.as_ref()) + else { + if pli.item_type.is_live() { + return Err(TuliproxError::RepositoryM3u("M3U catchup metadata missing".to_string())); + } return Err(TuliproxError::RepositoryM3u("M3U catchup requested for non-live stream".to_string())); }; - let Some(catchup) = live.catchup.as_ref() else { - return Err(TuliproxError::RepositoryM3u("M3U catchup metadata missing".to_string())); - }; let source_url = resolve_effective_source_url(pli, input)?; - let Some(resolved) = resolve_m3u_catchup_url(source_url.as_ref(), catchup, raw_query)? else { + let Some(resolved) = resolve_m3u_catchup_url(source_url.as_ref(), catchup.as_ref(), raw_query)? else { return Err(TuliproxError::RepositoryM3u("M3U catchup mode cannot be proxied".to_string())); }; @@ -523,6 +600,47 @@ async fn m3u_api_stream( return axum::http::StatusCode::BAD_REQUEST.into_response(); } + if let Some((req_virtual_id, archive)) = parse_flat_flussonic_archive(stream_req.stream_id) { + let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { + Ok(pli) => pli, + Err(err) => { + error!("Failed to read M3U item for native archive stream id {req_virtual_id}: {err}"); + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + }; + let Some(mode) = playlist_item_native_flussonic_mode(&pli) else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + if !native_archive_matches_mode(mode, &archive) { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + let input = try_option_bad_request!( + app_state.app_config.get_input_by_name(&pli.input_name), + true, + format!("Can't find input {} for target {target_name}, stream_id {}", pli.input_name, pli.virtual_id) + ); + let (resolved_pli, discriminator) = match apply_native_flussonic_archive(&pli, &input, &archive) { + Ok(result) => result, + Err(err) => { + debug!("Failed to resolve native M3U archive: {}", sanitize_sensitive_info(&err.to_string())); + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + }; + return m3u_api_stream_loaded( + user, + target, + fingerprint, + req_headers, + app_state, + resolved_pli, + input, + Some(archive.extension()), + Some(&discriminator), + ) + .await + .into_response(); + } + let (action_stream_id, stream_ext) = separate_number_and_remainder(stream_req.stream_id); let req_virtual_id: u32 = try_result_bad_request!(action_stream_id.trim().parse()); let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { @@ -572,6 +690,84 @@ async fn m3u_api_stream( .into_response() } +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn m3u_api_stream_nested( + user: Arc, + target: Arc, + fingerprint: &Fingerprint, + req_headers: &axum::http::HeaderMap, + app_state: &Arc, + stream_req: ApiStreamRequest<'_>, + file: &str, + raw_query: Option<&str>, +) -> impl IntoResponse + Send { + let _user_guard = app_state.app_config.file_locks.write_lock_str(&user.username).await; + if !target.has_output(TargetType::M3u) { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + let req_virtual_id: u32 = try_result_bad_request!(stream_req.stream_id.trim().parse()); + let pli = match m3u_get_item_for_stream_id(req_virtual_id, app_state, &target).await { + Ok(pli) => pli, + Err(err) => { + error!("Failed to read M3U item for nested stream id {req_virtual_id}: {err}"); + return axum::http::StatusCode::NOT_FOUND.into_response(); + } + }; + let input = try_option_bad_request!( + app_state.app_config.get_input_by_name(&pli.input_name), + true, + format!("Can't find input {} for target {}, stream_id {}", pli.input_name, target.name, pli.virtual_id) + ); + let stream_ext = extract_extension_from_url(file); + let resolved = if let Some(archive) = parse_flussonic_archive_file(file) { + let Some(mode) = playlist_item_native_flussonic_mode(&pli) else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + if !native_archive_matches_mode(mode, &archive) { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + apply_native_flussonic_archive(&pli, &input, &archive).map(|(item, discriminator)| { + (item, Some(discriminator)) + }) + } else if is_flussonic_live_file(file) { + let Some(mode) = playlist_item_native_flussonic_mode(&pli) else { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + }; + if !native_live_file_matches_mode(mode, file) { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + if has_m3u_catchup_marker(raw_query) { + resolve_loaded_m3u_catchup(&pli, &input, raw_query).map(|(item, discriminator)| { + (item, Some(discriminator)) + }) + } else { + Ok((pli, None)) + } + } else { + Err(TuliproxError::RepositoryM3u(format!("Unsupported nested M3U stream file: {file}"))) + }; + let (resolved_pli, archive_discriminator) = match resolved { + Ok(result) => result, + Err(err) => { + debug!("Failed to resolve nested M3U stream: {}", sanitize_sensitive_info(&err.to_string())); + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + }; + m3u_api_stream_loaded( + user, + target, + fingerprint, + req_headers, + app_state, + resolved_pli, + input, + stream_ext, + archive_discriminator.as_deref(), + ) + .await + .into_response() +} + async fn m3u_api_catchup( fingerprint: Fingerprint, req_headers: axum::http::HeaderMap, @@ -746,10 +942,50 @@ macro_rules! create_m3u_api_stream { }; } +macro_rules! create_nested_m3u_api_stream { + ($fn_name:ident, $context:expr) => { + async fn $fn_name( + fingerprint: Fingerprint, + req_headers: axum::http::HeaderMap, + axum::extract::RawQuery(raw_query): axum::extract::RawQuery, + axum::extract::Query(api_req): axum::extract::Query, + axum::extract::Path((username, password, stream_id, file)): axum::extract::Path<( + String, + String, + String, + String, + )>, + axum::extract::State(app_state): axum::extract::State>, + ) -> impl IntoResponse + Send { + let stream_req = ApiStreamRequest::from($context, &username, &password, &stream_id, ""); + let auth_status = app_state.app_config.get_auth_error_status(); + match m3u_api_stream_network_auth(&fingerprint, &app_state, &api_req, &stream_req) { + Ok((user, target)) => { + m3u_api_stream_nested( + user, + target, + &fingerprint, + &req_headers, + &app_state, + stream_req, + &file, + raw_query.as_deref(), + ) + .await + .into_response() + } + Err(err) => err.into_player_response(auth_status), + } + } + }; +} + create_m3u_api_stream!(m3u_api_live_stream_alt, ApiStreamContext::LiveAlt); create_m3u_api_stream!(m3u_api_live_stream, ApiStreamContext::Live); create_m3u_api_stream!(m3u_api_series_stream, ApiStreamContext::Series); create_m3u_api_stream!(m3u_api_movie_stream, ApiStreamContext::Movie); +create_nested_m3u_api_stream!(m3u_api_live_stream_alt_nested, ApiStreamContext::LiveAlt); +create_nested_m3u_api_stream!(m3u_api_live_stream_nested, ApiStreamContext::Live); macro_rules! register_m3u_api_stream { ($router:expr, [$(($path:expr, $fn_name:ident)),*]) => {{ @@ -784,6 +1020,21 @@ pub fn m3u_api_register() -> axum::Router> { (concat_path(storage_const::M3U_STREAM_PATH, "series"), m3u_api_series_stream) ] ); + router = router + .route( + &format!( + "/{}/{{username}}/{{password}}/{{stream_id}}/{{file}}", + storage_const::M3U_STREAM_PATH + ), + axum::routing::get(m3u_api_live_stream_alt_nested), + ) + .route( + &format!( + "/{}/{{username}}/{{password}}/{{stream_id}}/{{file}}", + concat_path(storage_const::M3U_STREAM_PATH, "live") + ), + axum::routing::get(m3u_api_live_stream_nested), + ); router.route( &format!("/{}/{{username}}/{{password}}/{{stream_id}}/{{resource}}", storage_const::M3U_RESOURCE_PATH), @@ -793,10 +1044,19 @@ pub fn m3u_api_register() -> axum::Router> { #[cfg(test)] mod tests { - use super::resolved_m3u_item_is_allowed; + use super::{ + apply_native_flussonic_archive, native_archive_matches_mode, native_live_file_matches_mode, + effective_playback_extension, playlist_item_native_flussonic_mode, resolve_m3u_catchup_properties, + resolved_m3u_item_is_allowed, + }; use crate::api::model::UserApiRequest; - use crate::model::ProxyUserCredentials; - use shared::model::{ClusterFlags, PlaylistItemType}; + use crate::model::{ConfigInput, ProxyUserCredentials}; + use crate::iptv::m3u::{parse_flussonic_archive_file, FlussonicArchiveKind}; + use shared::model::{ + CatchupProperties, ClusterFlags, LiveStreamProperties, M3uPlaylistItem, PlaylistItemType, StreamProperties, + }; + use shared::utils::Internable; + use std::{borrow::Cow, sync::Arc}; #[test] fn post_query_only_request_prefers_query_when_form_is_missing() { @@ -840,4 +1100,183 @@ mod tests { assert!(!resolved_m3u_item_is_allowed(&user, PlaylistItemType::Catchup)); } + + #[test] + fn catchup_uses_resolved_upstream_transport() { + assert_eq!( + effective_playback_extension( + PlaylistItemType::Catchup, + "http://provider/ch/timeshift_abs-1.ts", + Some(".m3u8") + ), + Some(".ts") + ); + assert_eq!( + effective_playback_extension( + PlaylistItemType::Catchup, + "http://provider/ch/timeshift_abs-1.m3u8", + Some(".ts") + ), + Some(".m3u8") + ); + assert_eq!( + effective_playback_extension(PlaylistItemType::Live, "http://provider/ch/index.m3u8", Some(".ts")), + Some(".ts") + ); + } + + #[test] + fn timeshift_only_live_items_derive_flussonic_catchup() -> Result<(), &'static str> { + let time_shift = Arc::::from("3"); + for item_type in [PlaylistItemType::Live, PlaylistItemType::LiveHls] { + let catchup = resolve_m3u_catchup_properties(item_type, &time_shift, None) + .ok_or("live timeshift item did not derive catchup")?; + assert_eq!(catchup.mode.as_deref(), Some("fs")); + assert_eq!(catchup.days.as_deref(), Some("3")); + } + Ok(()) + } + + #[test] + fn timeshift_only_non_live_item_does_not_derive_catchup() { + let time_shift = Arc::::from("3"); + assert!(resolve_m3u_catchup_properties(PlaylistItemType::Video, &time_shift, None).is_none()); + } + + #[test] + fn explicit_live_catchup_takes_precedence_over_timeshift_fallback() -> Result<(), &'static str> { + let explicit = CatchupProperties { + mode: Some(Arc::::from("shift")), + ..CatchupProperties::default() + }; + let properties = StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(explicit), + ..LiveStreamProperties::default() + })); + let time_shift = Arc::::from("3"); + + let catchup = resolve_m3u_catchup_properties(PlaylistItemType::LiveHls, &time_shift, Some(&properties)) + .ok_or("explicit catchup was not resolved")?; + + assert!(matches!(catchup, Cow::Borrowed(_))); + assert_eq!(catchup.mode.as_deref(), Some("shift")); + Ok(()) + } + + fn native_flussonic_item(mode: &str, url: &str) -> M3uPlaylistItem { + M3uPlaylistItem { + virtual_id: 59, + provider_id: "59".intern(), + name: "Channel".intern(), + chno: 0, + logo: "".intern(), + logo_small: "".intern(), + group: "Live".intern(), + title: "Channel".intern(), + parent_code: "".intern(), + audio_track: "".intern(), + time_shift: "3".intern(), + rec: "".intern(), + url: url.intern(), + epg_channel_id: None, + input_name: "input".intern(), + item_type: PlaylistItemType::Live, + t_stream_url: "".intern(), + t_resource_url: None, + t_catchup_source: None, + t_catchup_mode: None, + source_ordinal: 0, + additional_properties: Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + catchup_type: Some(mode.intern()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))), + input_stream_id: "59".intern(), + upstream_user_agent: None, + } + } + + #[test] + fn native_archive_resolution_handles_hls_and_ts() -> Result<(), shared::error::TuliproxError> { + let hls_item = native_flussonic_item("flussonic", "http://provider/ch/index.m3u8?token=abc"); + assert_eq!(playlist_item_native_flussonic_mode(&hls_item), Some("flussonic")); + let (hls_archive, discriminator) = apply_native_flussonic_archive( + &hls_item, + &ConfigInput::default(), + &FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".m3u8", + }, + )?; + assert_eq!(hls_archive.item_type, PlaylistItemType::Catchup); + assert_eq!(hls_archive.url.as_ref(), "http://provider/ch/index-1784898000-3600.m3u8?token=abc"); + assert_eq!(discriminator, "archive|1784898000|3600|.m3u8"); + + let ts_item = native_flussonic_item("flussonic-ts", "http://provider/ch/channel.ts?token=abc"); + let (ts_archive, _) = apply_native_flussonic_archive( + &ts_item, + &ConfigInput::default(), + &FlussonicArchiveKind::TimeshiftAbs { + start: "1784898000".to_string(), + extension: ".ts", + }, + )?; + assert_eq!(ts_archive.url.as_ref(), "http://provider/ch/timeshift_abs-1784898000.ts?token=abc"); + Ok(()) + } + + #[test] + fn native_archive_mode_is_not_inferred_from_timeshift_alone() { + let mut item = native_flussonic_item("append", "http://provider/ch/index.m3u8"); + item.time_shift = "7".intern(); + assert_eq!(playlist_item_native_flussonic_mode(&item), None); + } + + #[test] + fn native_paths_must_match_declared_transport() { + assert!(native_live_file_matches_mode("flussonic", "index.m3u8")); + assert!(!native_live_file_matches_mode("flussonic", "mpegts")); + assert!(native_live_file_matches_mode("flussonic-ts", "mpegts")); + assert!(!native_live_file_matches_mode("flussonic-ts", "index.m3u8")); + + assert!(native_archive_matches_mode( + "flussonic", + &FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".m3u8", + } + )); + assert!(!native_archive_matches_mode( + "flussonic-ts", + &FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".m3u8", + } + )); + assert!(native_archive_matches_mode( + "flussonic-ts", + &FlussonicArchiveKind::TimeshiftAbs { + start: "1784898000".to_string(), + extension: ".ts", + } + )); + } + + #[test] + fn nested_archive_file_is_parsed_and_checked_against_transport() { + let archive = parse_flussonic_archive_file("timeshift_abs-1784898000.ts"); + assert!(archive.as_ref().is_some_and(|value| native_archive_matches_mode("flussonic-ts", value))); + assert!(!archive.as_ref().is_some_and(|value| native_archive_matches_mode("flussonic", value))); + + for file in ["archive-1784898000-3600.ts", "timeshift_rel-120.ts"] { + let archive = parse_flussonic_archive_file(file); + assert!(archive.as_ref().is_some_and(|value| native_archive_matches_mode("flussonic-ts", value))); + assert!(!archive.as_ref().is_some_and(|value| native_archive_matches_mode("flussonic", value))); + } + } } diff --git a/backend/src/api/endpoints/v1_api.rs b/backend/src/api/endpoints/v1_api.rs index cd5934d87..4c7ece90d 100644 --- a/backend/src/api/endpoints/v1_api.rs +++ b/backend/src/api/endpoints/v1_api.rs @@ -222,6 +222,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }; app_state.connection_manager.add_connection(&addr).await; let registered = app_state diff --git a/backend/src/api/endpoints/v1_api_config.rs b/backend/src/api/endpoints/v1_api_config.rs index 6ae17a333..5f527a21b 100644 --- a/backend/src/api/endpoints/v1_api_config.rs +++ b/backend/src/api/endpoints/v1_api_config.rs @@ -2,10 +2,9 @@ use crate::{api::{ api_utils::{internal_server_error, try_unwrap_body}, config_file::ConfigFile, model::AppState, -}, auth::{permission_layer, verify_token, AuthBearer}, model::{validate_library_paths_from_dto, ApiProxyConfig, InputSource}, utils, utils::{ +}, auth::{permission_layer, verify_token, AuthBearer}, iptv::xtream::{get_xtream_stream_url_base, xtream_login}, model::{validate_library_paths_from_dto, ApiProxyConfig, InputSource}, utils, utils::{ persist_messaging_templates, prepare_sources_batch, prepare_users, read_api_proxy_file, request::download_text_content, - xtream::{get_xtream_stream_url_base, xtream_login}, }}; use axum::{ http::{header::IF_MATCH, HeaderMap, HeaderName, HeaderValue, StatusCode}, diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index a532d0212..4f51cbcb7 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -20,6 +20,7 @@ use crate::{ model::AppState, }, auth::{create_access_token, permission_layer, verify_access_token}, + iptv::{stalker::client::validate_public_playable_url, xtream}, model::{ parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, AppConfig, ConfigInput, ConfigInputFlags, ConfigInputOptions, EpgSource, EpgSourceType, IcsDummyPolicy, InputSource, @@ -35,8 +36,7 @@ use crate::{ utils::{ epg::get_input_raw_epg_file_path, file_exists_async, - network::stalker::client::validate_public_playable_url, - request, xtream, + request, }, }; use axum::{response::IntoResponse, Router}; diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index 6b5876622..e3107f8ba 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -524,6 +524,7 @@ fn from_programme( epg_id: &Arc, programme: &EpgProgramme, epg_processing_options: &EpgProcessingOptions, + has_archive: bool, ) -> ShortEpgDto { let (start_str, end_str, start_ts, stop_ts) = get_applied_epg_timeshift(programme, epg_processing_options); @@ -540,12 +541,13 @@ fn from_programme( stop_timestamp: stop_ts.to_string(), stream_id: Arc::clone(stream_id), now_playing: None, - has_archive: None, + has_archive: has_archive.then_some(1), } } const DEFAULT_SHORT_EPG_LIMIT: u32 = 4; +#[allow(clippy::too_many_arguments)] pub async fn serve_short_epg( app_state: &Arc, epg_path: &Path, @@ -554,6 +556,7 @@ pub async fn serve_short_epg( channel_id: &Arc, stream_id: Arc, limit: u32, + has_archive: bool, ) -> axum::response::Response { let lowercase_ids = target .options @@ -569,19 +572,13 @@ pub async fn serve_short_epg( if let Some(epg_channel) = get_epg_channel_by_storage_key(app_state, &storage_key, epg_path).await { let epg_processing_options = get_epg_processing_options(app_state, user, target); ShortEpgResultDto { - epg_listings: if limit > 0 { - epg_channel - .get_programme_with_limit(limit) - .iter() - .map(|p| from_programme(&stream_id, &response_channel_id, p, &epg_processing_options)) - .collect() - } else { - epg_channel - .programmes - .iter() - .map(|p| from_programme(&stream_id, &response_channel_id, p, &epg_processing_options)) - .collect() - }, + epg_listings: epg_channel + .get_programme_with_limit(limit) + .iter() + .map(|p| { + from_programme(&stream_id, &response_channel_id, p, &epg_processing_options, has_archive) + }) + .collect(), } } else { ShortEpgResultDto::default() @@ -1422,12 +1419,37 @@ mod tests { encrypt_secret: [0; 16], }; - let result = from_programme(&stream_id, &epg_id, &programme, &options); + let result = from_programme(&stream_id, &epg_id, &programme, &options, false); assert_eq!(result.epg_id.as_ref(), "example.channel"); assert_eq!(result.channel_id.as_ref(), "example.channel"); } + #[test] + fn from_programme_sets_has_archive_only_when_capability_true() { + let stream_id = "42".intern(); + let epg_id = "example.channel".intern(); + let programme = EpgProgramme::new_all( + 100, + 200, + Arc::clone(&epg_id), + Some("News".intern()), + None, + None, + ); + let options = EpgProcessingOptions { + rewrite_urls: false, + time_shift: EpgTimeShift::None, + encrypt_secret: [0; 16], + }; + + let archived = from_programme(&stream_id, &epg_id, &programme, &options, true); + let live_only = from_programme(&stream_id, &epg_id, &programme, &options, false); + + assert_eq!(archived.has_archive, Some(1)); + assert_eq!(live_only.has_archive, None); + } + #[test] fn lowercased_xmltv_display_name_remains_xml_escaped() { let display_name = lowercase_xmltv_text("CAFÉ NETWORK & ", true); @@ -1721,6 +1743,7 @@ mod tests { &request_id, "42".intern(), 4, + false, ) .await; let body = response_body_text(response).await; @@ -1740,6 +1763,7 @@ mod tests { &request_id, "42".intern(), 4, + false, ) .await; let disabled_body = response_body_text(disabled_response).await; diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index fcb785333..9c93b8dd9 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -6,13 +6,13 @@ use crate::{ api_utils, api_utils::{ admission_failure_response, coalesce_byte_stream, create_api_proxy_user, create_catchup_session_key, - create_playback_session_fingerprint, create_session_fingerprint, empty_json_response_as_array, - empty_json_response_as_object, force_provider_stream_response, get_session_reservation_ttl_secs, - get_user_target, get_user_target_by_credentials, internal_server_error, is_seek_request, - is_session_based_playback, is_stream_share_enabled, local_stream_response, redirect, redirect_response, - resource_response, separate_number_and_remainder, should_allow_exhausted_shared_reconnect, stream_response, - try_option_bad_request, try_result_bad_request, try_unwrap_body, RedirectParams, - resolve_initial_stalker_playback_url, + create_m3u_catchup_session_key, create_playback_session_fingerprint, create_session_fingerprint, + empty_json_response_as_array, empty_json_response_as_object, force_provider_stream_response, + get_session_reservation_ttl_secs, get_user_target, get_user_target_by_credentials, + internal_server_error, is_seekable_media_request, is_session_based_playback, is_stream_share_enabled, + local_stream_response, redirect, redirect_response, resource_response, separate_number_and_remainder, + should_allow_exhausted_shared_reconnect, stream_response, try_option_bad_request, try_result_bad_request, + try_unwrap_body, RedirectParams, resolve_initial_stalker_playback_url, }, endpoints::{ hls_api::{ @@ -27,6 +27,10 @@ use crate::{ }, }, auth::{verify_access_token, Fingerprint}, + iptv::{ + m3u::{is_xtream_m3u_catchup_supported, resolve_xtream_m3u_catchup_url, ResolvedM3uCatchup}, + xtream::{self, create_vod_info_from_item}, + }, model::{ xtream_mapping_option_from_target_options, Config, ConfigInput, ConfigInputFlags, ConfigTarget, InputSource, ProxyUserCredentials, @@ -36,8 +40,7 @@ use crate::{ xtream_get_collection_path, xtream_get_item_for_stream_id, xtream_load_rewrite_playlist, VirtualIdRecord, }, utils::{ - apply_timeshift, debug_if_enabled, file_exists_async, parse_timeshift, request, trace_if_enabled, xtream, - xtream::create_vod_info_from_item, + apply_timeshift, debug_if_enabled, file_exists_async, parse_timeshift, request, trace_if_enabled, }, }; use axum::{http::HeaderMap, response::IntoResponse}; @@ -54,7 +57,7 @@ use shared::{ error::TuliproxError, model::{ create_stream_channel_with_type, ConnectFailureReason, PlaylistEntry, PlaylistItemType, ProxyType, - ShortEpgResultDto, TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem, + ShortEpgResultDto, StreamProperties, TargetType, UserConnectionPermission, XtreamCluster, XtreamPlaylistItem, }, utils::{ deserialize_as_string, extract_extension_from_url, generate_provider_playlist_uuid, sanitize_sensitive_info, @@ -171,6 +174,11 @@ pub(in crate::api) fn get_xtream_player_api_stream_url( action_path: &str, fallback_url: &Arc, ) -> Option> { + // The resolved M3U archive URL is authoritative for timeshift requests. + if context == ApiStreamContext::Timeshift && input.input_type.is_m3u() && !fallback_url.is_empty() { + return Some(fallback_url.clone()); + } + if input.input_type.is_media_server() { return (!fallback_url.is_empty()).then(|| fallback_url.clone()); } @@ -357,6 +365,19 @@ async fn xtream_player_api_stream( ); } + let m3u_timeshift = if stream_req.context == ApiStreamContext::Timeshift { + try_result_bad_request!( + resolve_m3u_xtream_timeshift(&input, &pli, stream_req.action_path), + true, + format!("M3U Xtream timeshift rejected for stream {virtual_id}") + ) + } else { + None + }; + if let Some(resolved) = m3u_timeshift.as_ref() { + pli.url = resolved.url.as_str().intern(); + } + if pli.item_type.is_local() { let playback_session_token = create_session_fingerprint(fingerprint, &user.username, virtual_id, false); let user_session = @@ -415,14 +436,17 @@ async fn xtream_player_api_stream( } }; - let resolved_stream_ext = resolve_xtream_playback_extension(stream_ext, &pli); - let stream_ext = resolved_stream_ext.as_deref(); + let requested_extension = if m3u_timeshift.is_some() { + extract_extension_from_url(&pli.url).map(|ext| concat_string!(".", ext)) + } else { + resolve_xtream_playback_extension(stream_ext, &pli) + } + .unwrap_or_default(); + let stream_ext = (!requested_extension.is_empty()).then_some(requested_extension.as_str()); debug_if_enabled!( "ID chain for xtream endpoint: request_stream_id={} -> action_stream_id={action_stream_id} -> req_virtual_id={req_virtual_id} -> virtual_id={virtual_id}", stream_req.stream_id); - let requested_extension = stream_ext.unwrap_or_default(); - // Derive the playback extension from get_query_path so session semantics match // the actual path it will route. Falls back to requested_extension when empty. #[allow(clippy::needless_borrow, clippy::borrow_deref_ref)] @@ -430,7 +454,9 @@ async fn xtream_player_api_stream( #[allow(clippy::needless_borrow, clippy::borrow_deref_ref)] let playback_ext: &str = if playback_ext.is_empty() { &requested_extension } else { &playback_ext }; - let session_key = if item_type == PlaylistItemType::Catchup { + let session_key = if let Some(resolved) = m3u_timeshift.as_ref() { + create_m3u_catchup_session_key(fingerprint, &user.username, virtual_id, &resolved.discriminator) + } else if item_type == PlaylistItemType::Catchup { create_catchup_session_key(fingerprint, &user.username, virtual_id) } else { create_playback_session_fingerprint(fingerprint, &user.username, virtual_id, item_type, Some(playback_ext)) @@ -495,7 +521,7 @@ async fn xtream_player_api_stream( let stream_channel = create_stream_channel_with_type(target.id, &pli, item_type); - if session.virtual_id == virtual_id && is_seek_request(cluster, req_headers).await { + if session.virtual_id == virtual_id && is_seekable_media_request(cluster, req_headers, Some(playback_ext)) { // partial request means we are in reverse proxy mode, seek happened return force_provider_stream_response( fingerprint, @@ -707,6 +733,62 @@ fn resolve_xtream_playback_extension(stream_ext: Option<&str>, pli: &XtreamPlayl } } +// M3U timeshift must use the resolved archive URL instead of reconstructing an Xtream URL. +fn resolve_m3u_xtream_timeshift( + input: &ConfigInput, + item: &XtreamPlaylistItem, + action_path: &str, +) -> Result, TuliproxError> { + if !input.input_type.is_m3u() { + return Ok(None); + } + let (duration, start) = action_path.split_once('/').ok_or_else(|| { + TuliproxError::ApiXtream( + "M3U Xtream timeshift requires action_path in 'duration/start' form".to_string(), + ) + })?; + let props = item.additional_properties.as_ref().ok_or_else(|| { + TuliproxError::ApiXtream("M3U Xtream timeshift item has no stream properties".to_string()) + })?; + let StreamProperties::Live(live) = props else { + return Err(TuliproxError::ApiXtream( + "M3U Xtream timeshift requires a live stream".to_string(), + )); + }; + let catchup = live.catchup.as_ref().ok_or_else(|| { + TuliproxError::ApiXtream( + "M3U Xtream timeshift item has no catch-up metadata".to_string(), + ) + })?; + let resolved_source = input.resolve_url(&item.url)?; + let resolved = resolve_xtream_m3u_catchup_url(resolved_source.as_ref(), catchup, start, duration)?; + Ok(Some(resolved)) +} + +// Advertise M3U archives only when the timeshift bridge supports their template. +fn pli_supports_archive(app_state: &Arc, pli: &XtreamPlaylistItem) -> bool { + let Some(StreamProperties::Live(live)) = pli.additional_properties.as_ref() else { + return false; + }; + if live.tv_archive.unwrap_or(0) <= 0 { + return false; + } + let Some(input) = app_state.app_config.get_input_by_name(&pli.input_name) else { + return false; + }; + if input.input_type.is_m3u() { + let Some(catchup) = live.catchup.as_ref() else { + return false; + }; + match input.resolve_url(&pli.url) { + Ok(resolved) => is_xtream_m3u_catchup_supported(resolved.as_ref(), catchup), + Err(_) => false, + } + } else { + true + } +} + fn is_hls_playback_request(stream_ext: Option<&str>, pli: &XtreamPlaylistItem) -> bool { resolve_xtream_playback_extension(stream_ext, pli).as_deref() == Some(HLS_EXT) } @@ -770,6 +852,11 @@ pub(in crate::api) async fn xtream_player_api_stream_with_token( ) ); + // The token stream route has no action path from which to resolve an M3U archive. + if stream_req.context == ApiStreamContext::Timeshift && input.input_type.is_m3u() { + return axum::http::StatusCode::BAD_REQUEST.into_response(); + } + let user = create_api_proxy_user(app_state); if pli.item_type.is_local() { @@ -1243,6 +1330,7 @@ async fn xtream_get_short_epg( if let Ok(pli) = xtream_get_item_for_stream_id(virtual_id, app_state, target, None).await { let config = &app_state.app_config.config.load(); + let has_archive = pli_supports_archive(app_state, &pli); if let (Some(epg_path), Some(channel_id)) = ( get_epg_path_for_target_by_type(config, target, TargetType::Xtream), &pli.epg_channel_id, @@ -1256,6 +1344,7 @@ async fn xtream_get_short_epg( channel_id, stream_id.intern(), limit, + has_archive, ) .await; } @@ -1787,17 +1876,19 @@ pub fn xtream_api_register() -> axum::Router> { #[cfg(test)] mod tests { use super::{ - empty_stream_info_response, get_xtream_player_api_stream_url, is_hls_playback_request, - override_live_hls_extension, resolve_xtream_playback_extension, xtream_get_short_epg, ApiStreamContext, - XtreamApiTimeShiftRequest, + empty_stream_info_response, get_xtream_player_api_stream_url, resolve_m3u_xtream_timeshift, + is_hls_playback_request, override_live_hls_extension, resolve_xtream_playback_extension, + xtream_get_short_epg, xtream_player_api_stream, xtream_player_api_stream_with_token, ApiStreamContext, + ApiStreamRequest, XtreamApiTimeShiftRequest, }; + use crate::auth::Fingerprint; use crate::{ api::model::{ - create_test_app_state, PlaylistStorage, PlaylistXtreamStorage, UserApiRequest, + create_test_app_state, AppState, PlaylistStorage, PlaylistXtreamStorage, UserApiRequest, }, model::{ - Config, ConfigInput, ConfigTarget, Epg, IcsEpgSourceConfig, ProxyUserCredentials, TargetOutput, - XtreamTargetFlagsSet, XtreamTargetOutput, + Config, ConfigInput, ConfigTarget, Epg, IcsEpgSourceConfig, ProxyUserCredentials, SourcesConfig, + TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, }, processing::parser::ics::parse_ics_file_to_channel, repository::{ @@ -1806,12 +1897,12 @@ mod tests { }, }; use arc_swap::ArcSwapOption; - use axum::response::IntoResponse; + use axum::{http::HeaderMap, response::IntoResponse}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine}; - use shared::foundation::Filter; + use shared::{error::TuliproxError, foundation::Filter}; use shared::{ model::{ - ClusterFlags, InputType, PlaylistItemType, ProcessingOrder, StreamProperties, UUIDType, + ClusterFlags, InputType, PlaylistItemType, ProcessingOrder, ProxyUserStatus, StreamProperties, UUIDType, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, }, utils::Internable, @@ -1925,6 +2016,7 @@ mod tests { channel_no: 1, source_ordinal: 0, input_stream_id: "".intern(), + upstream_user_agent: None, } } @@ -2031,6 +2123,211 @@ mod tests { assert_eq!(decode_listing_text("description"), "Imported from ICS"); } + fn m3u_catchup_item( + name: &str, + input_name: &str, + url: &str, + catchup_source: Option<&str>, + ) -> XtreamPlaylistItem { + XtreamPlaylistItem { + virtual_id: 100, + provider_id: 0, + name: name.intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "Live".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: url.intern(), + epg_channel_id: Some("f1.calendar".intern()), + xtream_cluster: XtreamCluster::Live, + item_type: PlaylistItemType::Live, + category_id: 1, + input_name: input_name.intern(), + channel_no: 1, + source_ordinal: 0, + input_stream_id: "100".intern(), + additional_properties: Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { + name: name.intern(), + stream_id: 100, + tv_archive: Some(1), + tv_archive_duration: Some(7), + catchup: catchup_source.map(|src| shared::model::CatchupProperties { + mode: Some("flussonic".intern()), + source: Some(src.intern()), + ..shared::model::CatchupProperties::default() + }), + ..shared::model::LiveStreamProperties::default() + }))), + upstream_user_agent: None, + } + } + + async fn cache_xtream_test_item( + app_state: &Arc, + target: &Arc, + item: XtreamPlaylistItem, + input: Option, + ) { + if let Some(input) = input { + let sources = SourcesConfig { + inputs: vec![Arc::new(input)], + sources: vec![crate::model::ConfigSource { + inputs: vec![item.input_name.clone()], + targets: vec![target.clone()], + }], + ..SourcesConfig::default() + }; + app_state.app_config.sources.store(Arc::new(sources)); + } + let mut live = BPlusTree::new(); + live.insert(item.virtual_id, item.clone()); + app_state + .playlists + .cache_playlist( + &target.name, + PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { + live, + vod: BPlusTree::new(), + series: BPlusTree::new(), + })), + ) + .await; + let mut id_mapping = BPlusTree::new(); + id_mapping.insert( + item.virtual_id, + VirtualIdRecord::new(item.provider_id, item.virtual_id, PlaylistItemType::Live, 0, UUIDType::default()), + ); + app_state.playlists.cache_id_mapping(&target.name, id_mapping).await; + } + + async fn build_short_epg_app_state( + dir: &tempfile::TempDir, + item: XtreamPlaylistItem, + input: Option, + ) -> (Arc, Arc) { + let ics_path = dir.path().join("calendar.ics"); + let start = chrono::Utc::now() + chrono::Duration::days(1); + let stop = start + chrono::Duration::hours(1); + std::fs::write( + &ics_path, + format!( + concat!( + "BEGIN:VCALENDAR\r\n", + "VERSION:2.0\r\n", + "BEGIN:VEVENT\r\n", + "UID:f1-qualifying\r\n", + "DTSTART:{}\r\n", + "DTEND:{}\r\n", + "SUMMARY:Formula 1 Qualifying\r\n", + "DESCRIPTION:Imported from ICS\r\n", + "END:VEVENT\r\n", + "END:VCALENDAR\r\n", + ), + start.format("%Y%m%dT%H%M%SZ"), + stop.format("%Y%m%dT%H%M%SZ"), + ), + ) + .expect("write ICS fixture"); + let channel = parse_ics_file_to_channel( + &ics_path, + "f1.calendar".intern(), + Some("Formula 1".intern()), + &IcsEpgSourceConfig::default(), + ) + .await + .expect("parse ICS fixture"); + + let config = Config { + storage_dir: dir.path().to_string_lossy().into_owned(), + ..Config::default() + }; + let target = Arc::new(short_epg_target()); + let xtream_storage = xtream_get_storage_path(&config, &target.name).expect("xtream storage path"); + std::fs::create_dir_all(&xtream_storage).expect("create xtream storage"); + let epg_path = xtream_get_epg_file_path_for_target(&xtream_storage); + epg_write_file( + &target.name, + &Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(channel)], + }, + &epg_path, + &std::collections::HashMap::, Arc>::new(), + &shared::model::EpgOutputOptions::default(), + ) + .expect("write target EPG"); + + let app_state = create_test_app_state(config); + cache_xtream_test_item(&app_state, &target, item, input).await; + + (app_state, target) + } + + #[tokio::test] + async fn xtream_short_epg_emits_has_archive_for_m3u_with_bridge_template() { + let dir = tempdir().expect("temp dir"); + let input = m3u_timeshift_input(); + let item = m3u_catchup_item( + "Formula 1", + &input.name, + "channel/index.m3u8", + Some("http://provider.example/channel/video-{utc}-{duration}.m3u8"), + ); + let (app_state, target) = build_short_epg_app_state(&dir, item, Some(input)).await; + let mut user = ProxyUserCredentials::default(); + user.output_clusters = ClusterFlags::all(); + + let response = xtream_get_short_epg(&app_state, &user, &target, "100", 4).await.into_response(); + let body = response_body_text(response).await.expect("body"); + let json: serde_json::Value = serde_json::from_str(&body).expect("json"); + + assert_eq!(json["epg_listings"][0]["has_archive"], 1); + } + + #[tokio::test] + async fn xtream_short_epg_omits_has_archive_for_m3u_with_unsupported_catchup() { + let dir = tempdir().expect("temp dir"); + let input = m3u_timeshift_input(); + let item = m3u_catchup_item( + "Formula 1", + &input.name, + "channel/index.m3u8", + Some("http://provider.example/channel/${timestamp}.m3u8"), + ); + let (app_state, target) = build_short_epg_app_state(&dir, item, Some(input)).await; + let mut user = ProxyUserCredentials::default(); + user.output_clusters = ClusterFlags::all(); + + let body = response_body_text(xtream_get_short_epg(&app_state, &user, &target, "100", 4).await.into_response()) + .await + .expect("body"); + let json: serde_json::Value = serde_json::from_str(&body).expect("json"); + + assert!(json["epg_listings"][0].get("has_archive").is_none()); + } + + #[tokio::test] + async fn xtream_short_epg_emits_has_archive_for_native_xtream_input() { + let dir = tempdir().expect("temp dir"); + let mut input = m3u_timeshift_input(); + input.input_type = InputType::Xtream; + let item = m3u_catchup_item("Formula 1", &input.name, "channel/index.m3u8", None); + let (app_state, target) = build_short_epg_app_state(&dir, item, Some(input)).await; + let mut user = ProxyUserCredentials::default(); + user.output_clusters = ClusterFlags::all(); + + let body = response_body_text(xtream_get_short_epg(&app_state, &user, &target, "100", 4).await.into_response()) + .await + .expect("body"); + let json: serde_json::Value = serde_json::from_str(&body).expect("json"); + + assert_eq!(json["epg_listings"][0]["has_archive"], 1); + } + #[tokio::test] async fn empty_stream_info_response_uses_vod_object_and_list_shapes() -> Result<(), String> { assert_eq!(response_body_text(empty_stream_info_response(XtreamCluster::Video)).await?, "{}"); @@ -2076,6 +2373,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "813563".intern(), + upstream_user_agent: None, } } @@ -2275,4 +2573,202 @@ mod tests { assert_eq!(resolve_xtream_playback_extension(Some(".m3u8"), &pli).as_deref(), Some(".m3u8")); } + + fn m3u_timeshift_input() -> ConfigInput { + ConfigInput { + id: 0, + name: "m3u-flussonic".intern(), + input_type: InputType::M3u, + url: "http://provider.example".to_string(), + username: Some("alice".to_string()), + password: Some("secret".to_string()), + ..ConfigInput::default() + } + } + + fn m3u_timeshift_item() -> XtreamPlaylistItem { + XtreamPlaylistItem { + virtual_id: 100, + provider_id: 0, + name: "Live TV".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "Live".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "channel/index.m3u8".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Live, + additional_properties: Some(StreamProperties::Live(Box::new(shared::model::LiveStreamProperties { + name: "Live TV".intern(), + stream_id: 100, + tv_archive: Some(1), + tv_archive_duration: Some(7), + catchup: Some(shared::model::CatchupProperties { + mode: Some("flussonic".intern()), + source: Some("http://provider.example/channel/video-{utc}-{duration}.m3u8".intern()), + ..shared::model::CatchupProperties::default() + }), + ..shared::model::LiveStreamProperties::default() + }))), + item_type: PlaylistItemType::Live, + category_id: 1, + input_name: "m3u-flussonic".intern(), + channel_no: 1, + source_ordinal: 0, + input_stream_id: "100".intern(), + upstream_user_agent: None, + } + } + + #[test] + fn m3u_timeshift_resolver_returns_resolved_url_for_flussonic_template() { + let input = m3u_timeshift_input(); + let item = m3u_timeshift_item(); + + let resolved = resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:00-00") + .expect("resolver ok") + .expect("M3U input handled by bridge"); + + assert_eq!( + resolved.url, + "http://provider.example/channel/video-1704067200-3600.m3u8" + ); + assert!(!resolved.discriminator.is_empty()); + } + + #[test] + fn m3u_timeshift_resolver_distinguishes_time_windows() { + let input = m3u_timeshift_input(); + let item = m3u_timeshift_item(); + + let first = + resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:00-00").expect("first ok").expect("some"); + let second = + resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:01-00").expect("second ok").expect("some"); + + assert_ne!(first.url, second.url); + assert_ne!(first.discriminator, second.discriminator); + let fp = Fingerprint::new( + "fp".to_string(), + "127.0.0.1".to_string(), + "127.0.0.1:0".parse().unwrap(), + ); + assert_ne!( + crate::api::api_utils::create_m3u_catchup_session_key(&fp, "alice", 100, &first.discriminator), + crate::api::api_utils::create_m3u_catchup_session_key(&fp, "alice", 100, &second.discriminator), + ); + } + + #[test] + fn m3u_timeshift_resolver_rejects_missing_catchup_metadata() { + let mut item = m3u_timeshift_item(); + if let Some(StreamProperties::Live(live)) = item.additional_properties.as_mut() { + live.catchup = None; + } + let input = m3u_timeshift_input(); + + let err = resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:00-00") + .expect_err("missing catchup must error"); + assert!(matches!(err, TuliproxError::ApiXtream(_))); + } + + #[test] + fn m3u_timeshift_resolver_returns_none_for_non_m3u_input() { + let mut input = m3u_timeshift_input(); + input.input_type = InputType::Xtream; + let item = m3u_timeshift_item(); + + let resolved = resolve_m3u_xtream_timeshift(&input, &item, "60/2024-01-01:00-00") + .expect("non-M3U inputs are skipped, not errored"); + assert!(resolved.is_none(), "non-M3U input must be skipped by helper"); + } + + #[test] + fn m3u_timeshift_stream_url_uses_resolved_archive_for_credential_bearing_input() { + let input = m3u_timeshift_input(); + let resolved_url: Arc = + Arc::from("http://provider.example/channel/video-1704067200-3600.m3u8"); + + let url = + get_xtream_player_api_stream_url(&input, ApiStreamContext::Timeshift, "60/2024-01-01:00-00", &resolved_url) + .expect("resolved archive URL must win for M3U timeshift"); + + assert_eq!(url, resolved_url); + assert!(!url.contains("alice")); + assert!(!url.contains("secret")); + } + + #[tokio::test] + async fn expired_user_is_rejected_before_m3u_timeshift_is_resolved() -> Result<(), String> { + let dir = tempdir().map_err(|err| err.to_string())?; + let input = m3u_timeshift_input(); + let item = m3u_timeshift_item(); + let config = Config { + storage_dir: dir.path().to_string_lossy().into_owned(), + user_access_control: true, + ..Config::default() + }; + let app_state = create_test_app_state(config); + let target = Arc::new(short_epg_target()); + cache_xtream_test_item(&app_state, &target, item, Some(input)).await; + + let mut user = ProxyUserCredentials::default(); + user.username = "expired".to_string(); + user.status = Some(ProxyUserStatus::Expired); + user.output_clusters = ClusterFlags::all(); + let user = Arc::new(user); + let addr = "127.0.0.1:0".parse().map_err(|err: std::net::AddrParseError| err.to_string())?; + let fingerprint = Fingerprint::new("fp".to_string(), "127.0.0.1".to_string(), addr); + + let response = xtream_player_api_stream( + &fingerprint, + &HeaderMap::new(), + &app_state, + &UserApiRequest::default(), + ApiStreamRequest::from(ApiStreamContext::Timeshift, "expired", "", "100.ts", "invalid"), + Some((user, target)), + ) + .await + .into_response(); + + assert_eq!(response.status(), axum::http::StatusCode::BAD_GATEWAY); + Ok(()) + } + + #[tokio::test] + async fn xtream_token_stream_rejects_m3u_timeshift_with_bad_request() { + use crate::auth::create_access_token; + + let dir = tempdir().expect("temp dir"); + let config = Config { + storage_dir: dir.path().to_string_lossy().into_owned(), + ..Config::default() + }; + let app_state = create_test_app_state(config); + let target = Arc::new(short_epg_target()); + let input = m3u_timeshift_input(); + let item = m3u_timeshift_item(); + cache_xtream_test_item(&app_state, &target, item, Some(input)).await; + + let token = create_access_token(&app_state.app_config.access_token_secret, 60); + + let response = xtream_player_api_stream_with_token( + &Fingerprint::new("fp".to_string(), "127.0.0.1".to_string(), "127.0.0.1:0".parse().unwrap()), + &HeaderMap::new(), + &app_state, + target.id, + ApiStreamRequest::from_access_token( + ApiStreamContext::Timeshift, + &token, + "100.ts", + "", + ), + ) + .await + .into_response(); + + assert_eq!(response.status(), axum::http::StatusCode::BAD_REQUEST); + } } diff --git a/backend/src/api/model/active_user_manager.rs b/backend/src/api/model/active_user_manager.rs index 0b6a012ca..9055e0827 100644 --- a/backend/src/api/model/active_user_manager.rs +++ b/backend/src/api/model/active_user_manager.rs @@ -3504,6 +3504,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } @@ -3524,6 +3525,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } @@ -8701,6 +8703,7 @@ mod tests { shared: false, shared_joined_existing: None, shared_stream_id: None, technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }, user_agent: "ua".to_string(), country_code: None, diff --git a/backend/src/api/model/connection_manager.rs b/backend/src/api/model/connection_manager.rs index d6a6f678b..c5c127b8f 100644 --- a/backend/src/api/model/connection_manager.rs +++ b/backend/src/api/model/connection_manager.rs @@ -1264,6 +1264,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }; StreamInfo::new(shared::model::StreamInfoParams { uid: 0, diff --git a/backend/src/api/model/hls_cache/ids.rs b/backend/src/api/model/hls_cache/ids.rs index 49e502ce4..da1bcd1c4 100644 --- a/backend/src/api/model/hls_cache/ids.rs +++ b/backend/src/api/model/hls_cache/ids.rs @@ -15,6 +15,10 @@ pub struct HlsSessionKey { pub input_id: u16, /// Exact, non-empty origin/provider stream ID represented as a string. pub stream_ref: String, + /// Archive start timestamp; absent for live playback. + pub archive_reference: Option, + /// Opaque identity of the complete archive request; absent for live playback. + pub archive_identity: Option, } impl HlsSessionKey { @@ -23,10 +27,31 @@ impl HlsSessionKey { /// Callers must pass the origin/provider ID captured before target mapping, not /// a target-specific or virtual ID. pub fn new(input_id: u16, stream_ref: impl Into) -> Self { - Self { input_id, stream_ref: stream_ref.into() } + Self { input_id, stream_ref: stream_ref.into(), archive_reference: None, archive_identity: None } } - pub fn canonical(&self) -> String { format!("input:{}|hls|{}", self.input_id, self.stream_ref) } + pub const fn with_archive_reference(mut self, archive_reference: i64) -> Self { + self.archive_reference = Some(archive_reference); + self + } + + pub fn with_archive_identity(mut self, archive_identity: impl Into) -> Self { + self.archive_identity = Some(archive_identity.into()); + self + } + + pub fn canonical(&self) -> String { + self.archive_reference.map_or_else( + || format!("input:{}|hls|{}", self.input_id, self.stream_ref), + |timestamp| { + let base = format!("input:{}|hls|{}|archive|{timestamp}", self.input_id, self.stream_ref); + match self.archive_identity.as_ref() { + Some(identity) => format!("{base}|{identity}"), + None => base, + } + }, + ) + } pub fn stable_value(&self) -> String { self.canonical() } } @@ -66,6 +91,15 @@ mod tests { assert_ne!(first, different_stream); } + #[test] + fn archive_hls_session_key_changes_for_different_archive_request() { + let first = HlsSessionKey::new(7, "80510").with_archive_reference(1_784_898_000).with_archive_identity("first"); + let second = HlsSessionKey::new(7, "80510").with_archive_reference(1_784_898_000).with_archive_identity("second"); + + assert_ne!(first, second); + assert_ne!(first.stable_value(), second.stable_value()); + } + #[test] fn live_hls_session_key_preserves_alphanumeric_input_stream_id() { let key = HlsSessionKey::new(7, "m3u-channel_A42"); diff --git a/backend/src/api/model/hls_cache/lease.rs b/backend/src/api/model/hls_cache/lease.rs index 3fa2d6f86..bbd761d49 100644 --- a/backend/src/api/model/hls_cache/lease.rs +++ b/backend/src/api/model/hls_cache/lease.rs @@ -126,6 +126,8 @@ pub struct HlsAccessLease { pub pending_deadline: Option, pub valid_until_ms: u64, pub response_flag: Option, + pub epg_reference_ts: Option, + pub archive_origin_url: Option, } impl HlsAccessLease { @@ -162,9 +164,17 @@ impl HlsAccessLease { }), valid_until_ms: now_ms.saturating_add(valid_window_ms), response_flag: None, + epg_reference_ts: None, + archive_origin_url: None, } } + pub fn with_archive_playback(mut self, epg_reference_ts: Option, archive_origin_url: Option) -> Self { + self.epg_reference_ts = epg_reference_ts; + self.archive_origin_url = archive_origin_url; + self + } + pub const fn with_origin_acquire_policy(mut self, connection_kind: ConnectionKind, priority: i8) -> Self { self.origin_connection_kind = connection_kind; self.origin_priority = priority; @@ -739,6 +749,24 @@ mod tests { assert!(!lease_id.0.contains("session")); } + #[test] + fn archive_playback_context_is_opt_in() { + let live = lease(HlsAccessLeaseId("live".to_string()), "proxy-live", 1_000); + assert_eq!(live.epg_reference_ts, None); + assert_eq!(live.archive_origin_url, None); + + let archive = lease(HlsAccessLeaseId("archive".to_string()), "proxy-archive", 1_000) + .with_archive_playback( + Some(1_784_898_000), + Some("http://provider/channel/timeshift_abs-1784898000.m3u8".to_string()), + ); + assert_eq!(archive.epg_reference_ts, Some(1_784_898_000)); + assert_eq!( + archive.archive_origin_url.as_deref(), + Some("http://provider/channel/timeshift_abs-1784898000.m3u8") + ); + } + #[test] fn access_lease_activates_and_slides_validity() { let mut store = HlsAccessLeaseStore::default(); diff --git a/backend/src/api/model/hls_cache/origin.rs b/backend/src/api/model/hls_cache/origin.rs index d49c94994..4750069f6 100644 --- a/backend/src/api/model/hls_cache/origin.rs +++ b/backend/src/api/model/hls_cache/origin.rs @@ -32,6 +32,10 @@ pub struct HlsOriginSource { pub input_name: Arc, /// Exact, non-empty origin/provider stream ID captured before target mapping. pub stream_ref: String, + /// Archive start timestamp; absent for live playback. + pub archive_reference: Option, + /// Opaque identity of the complete archive request; absent for live playback. + pub archive_identity: Option, pub source_kind: HlsOriginSourceKind, } @@ -46,14 +50,49 @@ impl HlsOriginSource { stream_ref: impl Into, source_kind: HlsOriginSourceKind, ) -> Self { - Self { input_id, input_name, stream_ref: stream_ref.into(), source_kind } + Self { + input_id, + input_name, + stream_ref: stream_ref.into(), + archive_reference: None, + archive_identity: None, + source_kind, + } } pub fn from_session_key(key: &HlsSessionKey) -> Self { - Self::new(key.input_id, Arc::from(""), key.stream_ref.clone(), HlsOriginSourceKind::DirectMediaPlaylist) + let mut source = Self::new( + key.input_id, + Arc::from(""), + key.stream_ref.clone(), + HlsOriginSourceKind::DirectMediaPlaylist, + ); + source.archive_reference = key.archive_reference; + source.archive_identity.clone_from(&key.archive_identity); + source } - pub fn session_key(&self) -> HlsSessionKey { HlsSessionKey::new(self.input_id, self.stream_ref.clone()) } + pub const fn with_archive_reference(mut self, archive_reference: i64) -> Self { + self.archive_reference = Some(archive_reference); + self + } + + pub fn with_archive_request(mut self, archive_reference: i64, archive_url: &str) -> Self { + self.archive_reference = Some(archive_reference); + self.archive_identity = Some(blake3::hash(archive_url.as_bytes()).to_hex().to_string()); + self + } + + pub fn session_key(&self) -> HlsSessionKey { + let key = HlsSessionKey::new(self.input_id, self.stream_ref.clone()); + match self.archive_reference { + Some(timestamp) => match self.archive_identity.as_ref() { + Some(identity) => key.with_archive_reference(timestamp).with_archive_identity(identity), + None => key.with_archive_reference(timestamp), + }, + None => key, + } + } } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -967,6 +1006,28 @@ mod tests { assert_eq!(source.session_key(), key); } + #[test] + fn archive_source_keeps_origin_stream_ref_but_uses_distinct_session_key() { + let live = HlsOriginSource::new(7, Arc::from("input"), "80510", HlsOriginSourceKind::M3uMediaPlaylist); + let archive = live.clone().with_archive_reference(1_784_898_000); + + assert_eq!(live.stream_ref, "80510"); + assert_eq!(archive.stream_ref, "80510"); + assert_ne!(live.session_key(), archive.session_key()); + assert_eq!(archive.session_key().archive_reference, Some(1_784_898_000)); + } + + #[test] + fn archive_source_distinguishes_requests_with_same_start() { + let live = HlsOriginSource::new(7, Arc::from("input"), "80510", HlsOriginSourceKind::M3uMediaPlaylist); + let short = live + .clone() + .with_archive_request(1_784_898_000, "http://origin/80510-1784898000-1800.m3u8"); + let long = live.with_archive_request(1_784_898_000, "http://origin/80510-1784898000-3600.m3u8"); + + assert_ne!(short.session_key(), long.session_key()); + } + #[test] fn bound_account_acquire_error_rebind_policy_matches_concept() { assert!(HlsBoundAccountAcquireErrorKind::Missing.allows_rebind()); diff --git a/backend/src/api/model/hls_cache/playback.rs b/backend/src/api/model/hls_cache/playback.rs index 5f5941654..e7aa1f56a 100644 --- a/backend/src/api/model/hls_cache/playback.rs +++ b/backend/src/api/model/hls_cache/playback.rs @@ -27,6 +27,8 @@ pub struct HlsAccessContext { pub virtual_id: u32, pub lease_id: HlsAccessLeaseId, pub family_key: HlsPlaybackFamilyKey, + pub epg_reference_ts: Option, + pub archive_origin_url: Option, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -149,5 +151,7 @@ async fn validate_hls_access_lease_admission( virtual_id: lease.virtual_id, lease_id: lease.lease_id, family_key: lease.family_key, + epg_reference_ts: lease.epg_reference_ts, + archive_origin_url: lease.archive_origin_url, }) } diff --git a/backend/src/api/model/metadata_update_manager.rs b/backend/src/api/model/metadata_update_manager.rs index 67f436dcf..421668a2a 100644 --- a/backend/src/api/model/metadata_update_manager.rs +++ b/backend/src/api/model/metadata_update_manager.rs @@ -4252,6 +4252,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: Arc::from("42"), + upstream_user_agent: None, }; InputWorker::apply_pending_generic_probe_base(&batch, XtreamCluster::Video, 42, &mut item); diff --git a/backend/src/api/model/stream.rs b/backend/src/api/model/stream.rs index 4bb935560..dbf0ee870 100644 --- a/backend/src/api/model/stream.rs +++ b/backend/src/api/model/stream.rs @@ -19,7 +19,11 @@ pub type ProviderStreamInfo = Option<(ProviderStreamHeader, StatusCode, Option, ProviderStreamInfo); -pub type ProviderStreamFactoryResponse = (BoxedProviderStream, ProviderStreamInfo); +pub struct ProviderStreamFactoryResponse { + pub stream: BoxedProviderStream, + pub info: ProviderStreamInfo, + pub provider_session_headers: HashMap, +} /// Controls whether a provider stream preserves its origin representation or normalizes it to identity bytes. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -71,6 +75,7 @@ pub struct StreamDetails { pub provider_name: Option>, pub request_url: Option>, pub session_headers: Option>, + pub provider_session_headers: HashMap, pub grace_period: GracePeriodOptions, pub provider_grace_active: bool, pub disable_provider_grace: bool, @@ -93,6 +98,7 @@ impl Clone for StreamDetails { provider_name: self.provider_name.clone(), request_url: self.request_url.clone(), session_headers: self.session_headers.clone(), + provider_session_headers: self.provider_session_headers.clone(), grace_period: self.grace_period, provider_grace_active: self.provider_grace_active, disable_provider_grace: self.disable_provider_grace, @@ -112,6 +118,7 @@ impl StreamDetails { provider_name: None, request_url: None, session_headers: None, + provider_session_headers: HashMap::new(), grace_period: grace_period_options, provider_grace_active: false, disable_provider_grace: false, diff --git a/backend/src/api/model/streams/active_client_stream.rs b/backend/src/api/model/streams/active_client_stream.rs index 66bd9a23a..e9adf03f5 100644 --- a/backend/src/api/model/streams/active_client_stream.rs +++ b/backend/src/api/model/streams/active_client_stream.rs @@ -647,17 +647,23 @@ impl Stream for ActiveClientStream { ) .await { - Some(( - _stream, - Some((_headers, _status, _response_url, Some(custom_video_type))), - )) => { + Some(response) + if matches!( + response.info, + Some((_, _, _, Some(_))) + ) => { + let Some((_headers, _status, _response_url, Some(custom_video_type))) = + response.info + else { + return DeferredProviderOpenOutcome::Failed; + }; ActiveClientStreamState::mode_for_custom_video_type(custom_video_type) .map_or( DeferredProviderOpenOutcome::Failed, DeferredProviderOpenOutcome::Mode, ) } - Some((stream, _stream_info)) => DeferredProviderOpenOutcome::Stream(stream), + Some(response) => DeferredProviderOpenOutcome::Stream(response.stream), None => DeferredProviderOpenOutcome::Failed, } }); @@ -1593,6 +1599,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } @@ -1795,6 +1802,7 @@ mod tests { provider_name: Some(Arc::clone(provider_name)), request_url: Some("http://provider-1.example/live/1".intern()), session_headers: None, + provider_session_headers: HashMap::new(), grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, provider_grace_active: true, disable_provider_grace: false, @@ -2697,6 +2705,7 @@ mod tests { provider_name: Some(provider_name), request_url: Some("http://provider-1.example/live/2.ts".intern()), session_headers: None, + provider_session_headers: HashMap::new(), grace_period: GracePeriodOptions { period_millis: 100, timeout_secs: 0, hold_stream: true }, provider_grace_active: false, disable_provider_grace: false, diff --git a/backend/src/api/model/streams/provider_stream_factory.rs b/backend/src/api/model/streams/provider_stream_factory.rs index f030e2765..e93c9e370 100644 --- a/backend/src/api/model/streams/provider_stream_factory.rs +++ b/backend/src/api/model/streams/provider_stream_factory.rs @@ -2,7 +2,8 @@ use crate::{ api::{ api_utils::{get_headers_from_request, StreamOptions}, model::{ - create_channel_unavailable_stream, get_header_filter_for_item_type, get_response_headers, + create_channel_unavailable_stream, extract_hls_provider_session_headers, get_header_filter_for_item_type, + get_response_headers, log_hls_origin_content_coding, model_utils::{provider_response_headers, ProviderResponseHeaderError}, streams::{buffered_stream::BufferedStream, client_stream::ClientStream}, @@ -10,6 +11,7 @@ use crate::{ ProviderContentRepresentationMode, ProviderStreamFactoryResponse, StreamError, STREAM_IDLE_TIMEOUT, }, }, + iptv::stalker::client::validate_public_playable_url, model::{AppConfig, ConfigProvider, ReverseProxyDisabledHeaderConfig}, utils::{ content_coding::{ @@ -17,7 +19,6 @@ use crate::{ ContentCodingDetection, ContentCodingError, OutboundContentCodingPolicy, }, debug_if_enabled, - network::stalker::client::validate_public_playable_url, request::{ get_request_headers, is_safe_cross_origin_redirect_header, preview_request_diagnostics_for_logging, preview_request_target_for_logging, send_with_retry_and_provider_policy, @@ -27,7 +28,7 @@ use crate::{ use futures::{StreamExt, TryStreamExt}; use log::{debug, log_enabled, warn}; use reqwest::{ - header::{HeaderMap, CONTENT_RANGE, RANGE}, + header::{HeaderMap, HeaderValue, CONTENT_RANGE, RANGE}, StatusCode, }; use shared::{ @@ -86,7 +87,7 @@ pub struct ProviderStreamFactoryOptions { url: Url, headers: HeaderMap, default_user_agent: Option, - range_start_bytes: Option, + requested_range: Option, reconnect_flag: CancellationToken, provider: Option>, username: Option, @@ -143,18 +144,23 @@ impl ProviderStreamFactoryOptions { .map(ToString::to_string); let filter_header = get_header_filter_for_item_type(*item_type); let mut req_headers = get_headers_from_request(req_headers, &filter_header); - let requested_range = get_request_range_start_bytes(&req_headers); - req_headers.remove("range"); + let requested_range = + req_headers.remove(RANGE.as_str()).and_then(|value| HeaderValue::from_bytes(&value).ok()); let merged_input_headers = merge_provider_request_headers(*input_headers, *session_headers); // We merge configured input headers with the headers from the request. - let headers = get_request_headers( + let mut headers = get_request_headers( merged_input_headers.as_ref(), Some(&req_headers), *disabled_headers, *default_user_agent, ); + crate::utils::request::overlay_upstream_user_agent( + &mut headers, + stream_channel.as_ref().and_then(|channel| channel.upstream_user_agent.as_deref()), + *disabled_headers, + ); let default_user_agent = default_user_agent .and_then(|ua| { @@ -163,11 +169,6 @@ impl ProviderStreamFactoryOptions { }) .and_then(|ua| axum::http::header::HeaderValue::from_str(ua).ok()); let url = (*stream_url).clone(); - let range_start_bytes = if matches!(item_type, PlaylistItemType::Live | PlaylistItemType::LiveUnknown) { - requested_range - } else { - Some(requested_range.unwrap_or(0)) - }; let mut flags = ProviderStreamFactoryFlagsSet::new(); if stream_options.stream_retry { flags.set(ProviderStreamFactoryFlags::RetryEnabled); @@ -197,7 +198,7 @@ impl ProviderStreamFactoryOptions { url, headers, default_user_agent, - range_start_bytes, + requested_range, provider: None, username: username.map(ToString::to_string), client_ip: client_ip.map(ToString::to_string), @@ -263,7 +264,7 @@ impl ProviderStreamFactoryOptions { pub fn get_headers(&self) -> &HeaderMap { &self.headers } #[inline] - pub fn get_range_start_bytes(&self) -> Option { self.range_start_bytes } + pub fn get_requested_range(&self) -> Option<&HeaderValue> { self.requested_range.as_ref() } #[inline] pub fn should_continue(&self) -> bool { !self.reconnect_flag.is_cancelled() } @@ -468,23 +469,6 @@ fn should_wrap_provider_stream_in_buffer(stream_options: &ProviderStreamFactoryO && stream_options.is_buffer_enabled() } -fn get_request_range_start_bytes(req_headers: &HashMap>) -> Option { - // range header looks like bytes=1234-5566/2345345 or bytes=0- - if let Some(req_range) = req_headers.get(axum::http::header::RANGE.as_str()) { - if let Some(bytes_range) = req_range.strip_prefix(b"bytes=") { - if let Some(index) = bytes_range.iter().position(|&x| x == b'-') { - let start_bytes = &bytes_range[..index]; - if let Ok(start_str) = std::str::from_utf8(start_bytes) { - if let Ok(bytes_requested) = start_str.parse::() { - return Some(bytes_requested); - } - } - } - } - } - None -} - // fn get_host_and_optional_port(url: &Url) -> Option { // let host = url.host_str()?; // match url.port() { @@ -515,7 +499,7 @@ fn prepare_client( ) -> (reqwest::RequestBuilder, bool) { let original_url = stream_options.get_url(); let url = url_override.unwrap_or(original_url); - let range_start = stream_options.get_range_start_bytes(); + let requested_range = stream_options.get_requested_range(); let original_headers = stream_options.get_headers(); if log_enabled!(log::Level::Debug) { @@ -535,7 +519,7 @@ fn prepare_client( remove_sensitive_headers(&mut headers); } prepare_default_headers(&mut headers, stream_options); - let partial = prepare_partial_request_headers(&mut headers, stream_options, range_start); + let partial = prepare_partial_request_headers(&mut headers, requested_range); let content_coding_policy = match stream_options.content_representation() { ProviderContentRepresentationMode::PreserveOrigin => OutboundContentCodingPolicy::Inherit, ProviderContentRepresentationMode::Identity => OutboundContentCodingPolicy::Identity, @@ -592,21 +576,10 @@ fn prepare_default_headers(headers: &mut axum::http::HeaderMap, stream_options: } } -fn prepare_partial_request_headers( - headers: &mut HeaderMap, - stream_options: &ProviderStreamFactoryOptions, - range_start: Option, -) -> bool { - if let Some(range) = range_start { - if range > 0 || stream_options.was_range_requested() { - let range_header = format!("bytes={range}-"); - if let Ok(header_value) = axum::http::header::HeaderValue::from_str(&range_header) { - headers.insert(RANGE, header_value); - } - true - } else { - false - } +fn prepare_partial_request_headers(headers: &mut HeaderMap, requested_range: Option<&HeaderValue>) -> bool { + if let Some(range) = requested_range { + headers.insert(RANGE, range.clone()); + true } else { false } @@ -754,6 +727,7 @@ async fn prepare_provider_stream_response_with_context( context: ProviderStreamPreparationContext, idle_timeout: Duration, ) -> Result { + let provider_session_headers = extract_hls_provider_session_headers(response.headers()); match context.representation { ProviderContentRepresentationMode::PreserveOrigin => { if matches!(context.response_head_availability, ProviderResponseHeadAvailability::Unavailable) { @@ -765,7 +739,7 @@ async fn prepare_provider_stream_response_with_context( let headers = provider_response_headers(response.headers(), context.representation)?; let response_info = Some((headers, response.status(), Some(response.url().clone()), None)); let stream = response.bytes_stream().map_err(|error| StreamError::reqwest(&error)).boxed(); - Ok((stream, response_info)) + Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers }) } ProviderContentRepresentationMode::Identity => { let origin_status = response.status(); @@ -804,7 +778,7 @@ async fn prepare_provider_stream_response_with_context( let headers = provider_response_headers(&decoded.headers, context.representation)?; let response_info = Some((headers, decoded.status, Some(decoded.final_url), None)); let stream = ReaderStream::new(decoded.body).map_err(|error| provider_decoded_body_error(&error)).boxed(); - Ok((stream, response_info)) + Ok(ProviderStreamFactoryResponse { stream, info: response_info, provider_session_headers }) } } } @@ -899,7 +873,7 @@ async fn provider_stream_request( }); } let response = prepare_provider_stream_response_for_request(response, stream_options).await; - let (provider_stream, response_info) = match response { + let response = match response { Ok(response) => response, Err(error) => { let provider_error_class = error.provider_error_class(); @@ -913,10 +887,10 @@ async fn provider_stream_request( if log_enabled!(log::Level::Debug) { // Unfortunately, the HEAD request does not work, so we need this workaround. // We need some header information from the provider, we extract the necessary headers and forward them to the client - let message = format!("Provider response info: {response_info:?}"); + let message = format!("Provider response info: {:?}", response.info); debug!("{}", sanitize_sensitive_info(&message)); } - return Ok(Some((provider_stream, response_info))); + return Ok(Some(response)); } if status.is_client_error() { @@ -1067,7 +1041,7 @@ pub async fn create_provider_stream( stream_options: ProviderStreamFactoryOptions, ) -> Option { match get_provider_stream(app_state, client, &stream_options).await { - Ok(Some((init_stream, info))) => { + Ok(Some(ProviderStreamFactoryResponse { stream: init_stream, info, provider_session_headers })) => { if let Some((_headers, _status, _response_url, Some(custom_video_type))) = &info { let reason = match custom_video_type { CustomVideoStreamType::ChannelUnavailable => Some(ConnectFailureReason::ChannelUnavailable), @@ -1094,10 +1068,11 @@ pub async fn create_provider_stream( } else { stream }; - Some(( - ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()).boxed(), + Some(ProviderStreamFactoryResponse { + stream: ClientStream::new(stream, continue_signal.clone(), None, stream_options.get_url_as_str()).boxed(), info, - )) + provider_session_headers, + }) } Ok(None) => None, Err(failure) => { @@ -1115,7 +1090,11 @@ pub async fn create_provider_stream( &get_response_headers(stream_options.get_headers()), StatusCode::OK, ) { - return Some((boxed_provider_stream, response_info)); + return Some(ProviderStreamFactoryResponse { + stream: boxed_provider_stream, + info: response_info, + provider_session_headers: HashMap::new(), + }); } None } @@ -1261,7 +1240,7 @@ mod tests { ) { const IDENTITY_BODY: &[u8] = b"decoded redirect body"; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Available, @@ -1433,6 +1412,67 @@ mod tests { .await } + #[test] + fn provider_requests_preserve_complete_client_range_header() { + let stream_url = Url::parse("http://provider.example/movie").unwrap(); + let client = reqwest::Client::new(); + + for requested_range in [ + "bytes=100-199", + "bytes=100-", + "bytes=-100", + "bytes=0-0", + "bytes=0-0,200-299", + ] { + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::RANGE, requested_range.parse().unwrap()); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + None, + None, + ); + + let (request, partial) = + prepare_client(&client, &options, None, ProviderRequestCredentialState::OriginalOrigin); + let request = request.build().unwrap(); + + assert!(partial, "Range request was not recognized: {requested_range}"); + assert_eq!( + request.headers().get(reqwest::header::RANGE).and_then(|value| value.to_str().ok()), + Some(requested_range) + ); + } + } + + #[test] + fn client_range_takes_precedence_over_configured_provider_range() { + let stream_url = Url::parse("http://provider.example/movie").unwrap(); + let mut req_headers = HeaderMap::new(); + req_headers.insert(reqwest::header::RANGE, "bytes=100-199".parse().unwrap()); + let input_headers = HashMap::from([("Range".to_string(), "bytes=0-".to_string())]); + let options = test_options( + ProviderContentRepresentationMode::PreserveOrigin, + &stream_url, + &req_headers, + Some(&input_headers), + None, + ); + + let request = prepare_client( + &reqwest::Client::new(), + &options, + None, + ProviderRequestCredentialState::OriginalOrigin, + ) + .0 + .build() + .unwrap(); + + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=100-199"); + } + #[test] fn test_provider_stream_factory_options_range_logic() { let addr = "127.0.0.1:8080".parse().unwrap(); @@ -1461,7 +1501,7 @@ mod tests { content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(!options.was_range_requested()); - assert_eq!(options.get_range_start_bytes(), Some(0)); // Should track range start even if not requested + assert!(options.get_requested_range().is_none()); // Case 2: VOD, range requested req_headers.insert("Range", "bytes=100-".parse().unwrap()); @@ -1483,7 +1523,7 @@ mod tests { content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(options.was_range_requested()); - assert_eq!(options.get_range_start_bytes(), Some(100)); + assert_eq!(options.get_requested_range().and_then(|value| value.to_str().ok()), Some("bytes=100-")); // Case 3: Live, no initial range requested let req_headers = HeaderMap::new(); @@ -1505,7 +1545,7 @@ mod tests { content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(!options.was_range_requested()); - assert_eq!(options.get_range_start_bytes(), None); // Should NOT track range start + assert!(options.get_requested_range().is_none()); // Case 4: Live, range requested (should be stripped) let mut req_headers = HeaderMap::new(); @@ -1528,7 +1568,7 @@ mod tests { content_representation: ProviderContentRepresentationMode::PreserveOrigin, }); assert!(!options.was_range_requested()); // Stripped by filter - assert_eq!(options.get_range_start_bytes(), None); + assert!(options.get_requested_range().is_none()); } #[test] @@ -1650,6 +1690,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: Some("Channel-UA".intern()), }), connect_failure_stage: Some(FailureStage::ProviderOpen), content_representation: ProviderContentRepresentationMode::PreserveOrigin, @@ -1662,6 +1703,7 @@ mod tests { assert_eq!(info.provider.as_ref(), "provider-a"); assert_eq!(info.channel.input_name.as_ref(), "input-a"); assert_eq!(info.channel.virtual_id, 77); + assert_eq!(options.headers.get(reqwest::header::USER_AGENT).and_then(|value| value.to_str().ok()), Some("Channel-UA")); } #[test] @@ -1717,12 +1759,38 @@ mod tests { ); } + #[tokio::test] + async fn provider_stream_factory_captures_session_cookies_separately() { + let (response, _) = local_response( + StatusCode::OK, + &[("content-type", "application/vnd.apple.mpegurl"), ("set-cookie", "sid=abc; Path=/")], + b"#EXTM3U\nsegment.ts\n".to_vec(), + ) + .await; + + let result = prepare_provider_stream_response( + response, + ProviderContentRepresentationMode::Identity, + ProviderResponseHeadAvailability::Available, + ) + .await; + + assert!(result.is_ok()); + if let Ok(response) = result { + assert_eq!(response.provider_session_headers.get("cookie").map(String::as_str), Some("sid=abc")); + assert!(response + .info + .as_ref() + .is_some_and(|(headers, _, _, _)| headers.iter().all(|(name, _)| !name.eq_ignore_ascii_case("set-cookie")))); + } + } + #[test] fn cross_origin_provider_requests_keep_only_redirect_safe_headers() { let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); let mut req_headers = HeaderMap::new(); req_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); - req_headers.insert(reqwest::header::RANGE, "bytes=17-".parse().unwrap()); + req_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); req_headers.insert(reqwest::header::USER_AGENT, "test-agent".parse().unwrap()); req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); @@ -1742,7 +1810,7 @@ mod tests { let mut safe_headers = HeaderMap::new(); safe_headers.insert(reqwest::header::ACCEPT_ENCODING, "gzip".parse().unwrap()); - safe_headers.insert(reqwest::header::RANGE, "bytes=17-".parse().unwrap()); + safe_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); let safe_options = test_options(ProviderContentRepresentationMode::PreserveOrigin, &stream_url, &safe_headers, None, None); assert!(!provider_headers_require_manual_redirects(safe_options.get_headers())); @@ -1774,7 +1842,7 @@ mod tests { assert!(!request.headers().contains_key(name), "cross-origin request retained {name}"); } assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "gzip"); - assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-"); + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); assert_eq!(request.headers()[reqwest::header::USER_AGENT], "test-agent"); } } @@ -1811,7 +1879,7 @@ mod tests { let stream_url = Url::parse("http://provider-a.example/live/segment.ts").unwrap(); let mut req_headers = HeaderMap::new(); req_headers.insert(reqwest::header::ACCEPT_ENCODING, "br".parse().unwrap()); - req_headers.insert(reqwest::header::RANGE, "bytes=17-".parse().unwrap()); + req_headers.insert(reqwest::header::RANGE, "bytes=17-31".parse().unwrap()); req_headers.insert(reqwest::header::AUTHORIZATION, "Bearer client".parse().unwrap()); req_headers.insert(reqwest::header::COOKIE, "client=1".parse().unwrap()); let input_headers = HashMap::from([("Accept-Encoding".to_string(), "gzip".to_string())]); @@ -1838,7 +1906,7 @@ mod tests { .build() .unwrap(); assert_eq!(request.headers()[reqwest::header::ACCEPT_ENCODING], "identity"); - assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-"); + assert_eq!(request.headers()[reqwest::header::RANGE], "bytes=17-31"); if target == Some(&cross_origin) { assert!(!request.headers().contains_key(reqwest::header::AUTHORIZATION)); assert!(!request.headers().contains_key(reqwest::header::COOKIE)); @@ -1937,7 +2005,7 @@ mod tests { let encoded = encode(IDENTITY_BODY, TestEncoding::Gzip).await; let (response, _) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], encoded).await; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Unavailable, @@ -2039,7 +2107,7 @@ mod tests { let (response, _) = local_response(StatusCode::OK, &[("Content-Encoding", content_encoding)], encoded).await; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Available, @@ -2060,7 +2128,7 @@ mod tests { let body = Bytes::from_static(b"\x1f\x8bnot-an-encoded-key"); let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; - let (stream, _) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Available, @@ -2078,7 +2146,7 @@ mod tests { local_response(StatusCode::PARTIAL_CONTENT, &[("Content-Range", "bytes 0-2/10")], PARTIAL_BODY.to_vec()) .await; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Available, @@ -2102,7 +2170,7 @@ mod tests { local_response(StatusCode::OK, &[("Content-Encoding", "gzip"), ("Content-Range", "bytes 0-9/10")], encoded) .await; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Available, @@ -2173,7 +2241,7 @@ mod tests { ) .await; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::PreserveOrigin, ProviderResponseHeadAvailability::Available, @@ -2202,7 +2270,7 @@ mod tests { ) .await; - let (stream, info) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, info, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::PreserveOrigin, ProviderResponseHeadAvailability::Available, @@ -2223,7 +2291,7 @@ mod tests { let body = Bytes::from_static(b"\x1f\x8bopaque non-hls bytes"); let (response, _) = local_response(StatusCode::OK, &[], body.to_vec()).await; - let (stream, _) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::PreserveOrigin, ProviderResponseHeadAvailability::Available, @@ -2240,7 +2308,7 @@ mod tests { truncated.truncate(truncated.len().saturating_sub(5)); let (response, requests) = local_response(StatusCode::OK, &[("Content-Encoding", "gzip")], truncated).await; - let (stream, _) = prepare_provider_stream_response( + let ProviderStreamFactoryResponse { stream, .. } = prepare_provider_stream_response( response, ProviderContentRepresentationMode::Identity, ProviderResponseHeadAvailability::Available, diff --git a/backend/src/api/model/streams/shared_stream_manager.rs b/backend/src/api/model/streams/shared_stream_manager.rs index 2fca85aca..ad6c6a078 100644 --- a/backend/src/api/model/streams/shared_stream_manager.rs +++ b/backend/src/api/model/streams/shared_stream_manager.rs @@ -1070,6 +1070,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, } } diff --git a/backend/src/api/panel_api.rs b/backend/src/api/panel_api.rs index 7b7fab606..f4ea2e202 100644 --- a/backend/src/api/panel_api.rs +++ b/backend/src/api/panel_api.rs @@ -4070,6 +4070,7 @@ pub fn create_panel_api_provisioning_stream_details( provider_name, request_url: None, session_headers: None, + provider_session_headers: HashMap::new(), grace_period: *grace_period_options, provider_grace_active: false, disable_provider_grace: true, @@ -4097,6 +4098,7 @@ pub fn create_panel_api_provisioning_stream_details( provider_name, request_url: None, session_headers: None, + provider_session_headers: HashMap::new(), grace_period: *grace_period_options, provider_grace_active: false, disable_provider_grace: true, diff --git a/backend/src/api/serve.rs b/backend/src/api/serve.rs index 24904a4ce..0dbacba59 100644 --- a/backend/src/api/serve.rs +++ b/backend/src/api/serve.rs @@ -265,6 +265,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }; let upstream = PendingDropProbeStream { dropped: Arc::clone(&state.upstream_dropped), diff --git a/backend/src/iptv/m3u/archive.rs b/backend/src/iptv/m3u/archive.rs new file mode 100644 index 000000000..a1f6995ff --- /dev/null +++ b/backend/src/iptv/m3u/archive.rs @@ -0,0 +1,293 @@ +use url::Url; + +pub const FLUSSONIC_HLS_LIVE_FILE: &str = "index.m3u8"; +pub const FLUSSONIC_TS_LIVE_FILE: &str = "mpegts"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FlussonicArchiveKind { + Archive { start: String, duration: String, extension: &'static str }, + TimeshiftAbs { start: String, extension: &'static str }, + TimeshiftRel { ago: String, extension: &'static str }, +} + +impl FlussonicArchiveKind { + pub fn discriminator(&self) -> String { + match self { + Self::Archive { start, duration, extension } => { + format!("archive|{start}|{duration}|{extension}") + } + Self::TimeshiftAbs { start, extension } => format!("timeshift_abs|{start}|{extension}"), + Self::TimeshiftRel { ago, extension } => format!("timeshift_rel|{ago}|{extension}"), + } + } + + pub fn extension(&self) -> &'static str { + match self { + Self::Archive { extension, .. } + | Self::TimeshiftAbs { extension, .. } + | Self::TimeshiftRel { extension, .. } => extension, + } + } +} + +fn strip_media_suffix(value: &str) -> &str { + value + .strip_suffix(".m3u8") + .or_else(|| value.strip_suffix(".M3U8")) + .or_else(|| value.strip_suffix(".ts")) + .or_else(|| value.strip_suffix(".TS")) + .unwrap_or(value) +} + +fn split_media_suffix(value: &str) -> (&str, &'static str) { + if let Some(stem) = value.strip_suffix(".m3u8") { + (stem, ".m3u8") + } else if let Some(stem) = value.strip_suffix(".ts") { + (stem, ".ts") + } else { + (value, ".m3u8") + } +} + +fn is_valid_start_duration(start: &str, duration: &str) -> bool { + start.parse::().is_ok() + && (duration.eq_ignore_ascii_case("now") || duration.parse::().is_ok()) +} + +fn split_start_duration(value: &str) -> Option<(&str, &str)> { + let (start, duration) = value.split_once('-')?; + if !is_valid_start_duration(start, duration) { + return None; + } + Some((start, duration)) +} + +pub fn parse_flat_flussonic_archive(file: &str) -> Option<(u32, FlussonicArchiveKind)> { + if file.contains('/') + || !std::path::Path::new(file).extension().is_some_and(|ext| ext.eq_ignore_ascii_case("m3u8")) + { + return None; + } + let stem = strip_media_suffix(file.trim()); + let mut parts = stem.rsplitn(3, '-'); + let duration = parts.next()?; + let start = parts.next()?; + let virtual_id = parts.next()?.parse::().ok()?; + if !is_valid_start_duration(start, duration) { + return None; + } + Some(( + virtual_id, + FlussonicArchiveKind::Archive { + start: start.to_string(), + duration: duration.to_string(), + extension: ".m3u8", + }, + )) +} + +pub fn parse_flussonic_archive_file(file: &str) -> Option { + let file = file.trim(); + if file.is_empty() || file.contains('/') { + return None; + } + let lower = file.to_ascii_lowercase(); + for prefix in ["archive-", "index-", "video-", "mono-"] { + if let Some(value) = lower.strip_prefix(prefix) { + let (value, extension) = split_media_suffix(value); + let (start, duration) = split_start_duration(value)?; + return Some(FlussonicArchiveKind::Archive { + start: start.to_string(), + duration: duration.to_string(), + extension, + }); + } + } + if let Some(value) = lower.strip_prefix("timeshift_abs-") { + let (start, extension) = if let Some(start) = value.strip_suffix(".m3u8") { + (start, ".m3u8") + } else { + (value.strip_suffix(".ts")?, ".ts") + }; + start.parse::().ok()?; + return Some(FlussonicArchiveKind::TimeshiftAbs { start: start.to_string(), extension }); + } + if let Some(value) = lower.strip_prefix("timeshift_rel-") { + let (ago, extension) = split_media_suffix(value); + ago.parse::().ok()?; + return Some(FlussonicArchiveKind::TimeshiftRel { ago: ago.to_string(), extension }); + } + None +} + +pub fn is_flussonic_live_file(file: &str) -> bool { + ["index.m3u8", "video.m3u8", "mono.m3u8", "mpegts", "index.ts", "video.ts", "mono.ts"] + .iter() + .any(|supported_file| file.trim().eq_ignore_ascii_case(supported_file)) +} + +fn provider_playlist_stem(provider_url: &str) -> Option<&'static str> { + let parsed = Url::parse(provider_url).ok()?; + let file = parsed.path_segments()?.next_back()?; + if file.eq_ignore_ascii_case("index.m3u8") || file.eq_ignore_ascii_case("index.ts") { + Some("index") + } else if file.eq_ignore_ascii_case("video.m3u8") || file.eq_ignore_ascii_case("video.ts") { + Some("video") + } else if file.eq_ignore_ascii_case("mono.m3u8") || file.eq_ignore_ascii_case("mono.ts") { + Some("mono") + } else { + None + } +} + +pub fn build_provider_flussonic_archive_url( + provider_url: &str, + archive: &FlussonicArchiveKind, +) -> Option { + let new_file = match archive { + FlussonicArchiveKind::Archive { start, duration, extension } => { + format!("{}-{start}-{duration}{extension}", provider_playlist_stem(provider_url).unwrap_or("archive")) + } + FlussonicArchiveKind::TimeshiftAbs { start, extension } => { + format!("timeshift_abs-{start}{extension}") + } + FlussonicArchiveKind::TimeshiftRel { ago, extension } => format!("timeshift_rel-{ago}{extension}"), + }; + let mut parsed = Url::parse(provider_url).ok()?; + parsed.path_segments_mut().ok()?.pop().push(&new_file); + Some(parsed.into()) +} + +pub fn flussonic_proxy_live_file(player_mode: &str) -> &'static str { + if player_mode.eq_ignore_ascii_case("flussonic-ts") { + FLUSSONIC_TS_LIVE_FILE + } else { + FLUSSONIC_HLS_LIVE_FILE + } +} + +#[cfg(test)] +mod tests { + use super::{ + build_provider_flussonic_archive_url, parse_flat_flussonic_archive, parse_flussonic_archive_file, + FlussonicArchiveKind, + }; + + #[test] + fn parses_flat_tivimate_hls_archive() -> Result<(), &'static str> { + let (virtual_id, archive) = parse_flat_flussonic_archive("59-1784898000-3600.m3u8") + .ok_or("flat archive path was not parsed")?; + assert_eq!(virtual_id, 59); + assert_eq!( + archive, + FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".m3u8", + } + ); + assert!(parse_flat_flussonic_archive("59.m3u8").is_none()); + assert!(parse_flat_flussonic_archive("59-1784898000-3600.ts").is_none()); + + let uppercase = parse_flat_flussonic_archive("59-1784898000-3600.M3U8") + .ok_or("uppercase flat archive path was not parsed")?; + assert_eq!(uppercase, (virtual_id, archive)); + Ok(()) + } + + #[test] + fn archive_discriminator_includes_transport() { + let hls = FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".m3u8", + }; + let ts = FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".ts", + }; + + assert_ne!(hls.discriminator(), ts.discriminator()); + } + + #[test] + fn parses_nested_hls_and_ts_archive_files() { + for file in [ + "archive-1784898000-3600.m3u8", + "index-1784898000-3600.m3u8", + "video-1784898000-3600.m3u8", + "mono-1784898000-now.m3u8", + ] { + assert!(matches!( + parse_flussonic_archive_file(file), + Some(FlussonicArchiveKind::Archive { .. }) + )); + } + assert_eq!( + parse_flussonic_archive_file("timeshift_abs-1784898000.ts"), + Some(FlussonicArchiveKind::TimeshiftAbs { + start: "1784898000".to_string(), + extension: ".ts", + }) + ); + + for file in ["archive-1784898000-3600.ts", "timeshift_rel-120.ts"] { + let archive = parse_flussonic_archive_file(file).expect("TS archive path should parse"); + let url = build_provider_flussonic_archive_url("http://cdn.example/ch/index.ts", &archive) + .expect("TS provider archive URL should build"); + assert!(std::path::Path::new(&url) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("ts")), "requested TS extension was not preserved: {url}"); + } + } + + #[test] + fn rejects_malformed_native_archive_files() { + for file in [ + "index.m3u8", + "index-not-a-time-3600.m3u8", + "archive-1784898000-invalid.m3u8", + "timeshift_abs-x.ts", + "timeshift_abs--1.ts", + "timeshift_rel--120.m3u8", + "../../archive-1784898000-3600.m3u8", + ] { + assert!(parse_flussonic_archive_file(file).is_none(), "accepted malformed file: {file}"); + } + } + + #[test] + fn identifies_only_supported_native_live_files() { + for file in ["index.m3u8", "video.m3u8", "mono.m3u8", "mpegts", "index.ts"] { + assert!(super::is_flussonic_live_file(file)); + } + assert!(!super::is_flussonic_live_file("archive-1784898000-3600.m3u8")); + assert!(!super::is_flussonic_live_file("segment.ts")); + } + + #[test] + fn builds_hls_and_ts_provider_archive_urls_without_losing_query() -> Result<(), &'static str> { + let hls = build_provider_flussonic_archive_url( + "http://cdn.example/ch/index.m3u8?token=abc", + &FlussonicArchiveKind::Archive { + start: "1784898000".to_string(), + duration: "3600".to_string(), + extension: ".m3u8", + }, + ) + .ok_or("HLS provider archive URL was not built")?; + assert_eq!(hls, "http://cdn.example/ch/index-1784898000-3600.m3u8?token=abc"); + + let ts = build_provider_flussonic_archive_url( + "http://cdn.example/ch/channel.ts?token=abc", + &FlussonicArchiveKind::TimeshiftAbs { + start: "1784898000".to_string(), + extension: ".ts", + }, + ) + .ok_or("TS provider archive URL was not built")?; + assert_eq!(ts, "http://cdn.example/ch/timeshift_abs-1784898000.ts?token=abc"); + Ok(()) + } +} diff --git a/backend/src/iptv/m3u/catchup.rs b/backend/src/iptv/m3u/catchup.rs new file mode 100644 index 000000000..d254fde7f --- /dev/null +++ b/backend/src/iptv/m3u/catchup.rs @@ -0,0 +1,1090 @@ +use super::{encode_m3u_catchup_token, M3uCatchupToken}; +use chrono::{DateTime, NaiveDateTime}; +use shared::{ + error::TuliproxError, + model::CatchupProperties, + utils::{short_hash, Internable}, +}; +use std::{borrow::Cow, sync::Arc}; +use url::Url; + +pub const M3U_CATCHUP_ROUTE_PREFIX: &str = "m3u-catchup"; +pub const M3U_CATCHUP_MARKER: &str = "tuliprox-catchup"; +const COLLECTOR_PREFIX: &str = "v"; +const M3U_APPEND_TEMPLATE: &str = "?utc={utc}&lutc={lutc}"; +const M3U_CATCHUP_PARAM_UTC: &str = "utc"; +const M3U_CATCHUP_PARAM_LUTC: &str = "lutc"; +const XC_START_TEMPLATE: &str = "{Y}-{m}-{d}:{H}-{M}-{S}"; +const FLUSSONIC_UTC_SENTINEL: &str = "__TULIPROX_M3U_CATCHUP_UTC__"; +const XTREAM_BRIDGE_START_FORMATS: [&str; 4] = [ + "%Y-%m-%d %H:%M", + "%Y-%m-%d:%H-%M", + "%Y-%m-%d:%H:%M", + "%Y-%m-%d-%H-%M", +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct M3uCatchupRewrite { + pub mode: Arc, + pub source: Arc, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedM3uCatchup { + pub url: String, + pub discriminator: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum TemplateSegment { + Literal(String), + Placeholder(String), +} + +fn mode_alias(mode: &str) -> &str { + if mode.eq_ignore_ascii_case("append") { + "append" + } else if mode.eq_ignore_ascii_case("default") { + "default" + } else if mode.eq_ignore_ascii_case("shift") { + "shift" + } else if mode.eq_ignore_ascii_case("xc") { + "xc" + } else if ["fs", "flussonic", "flussonic-hls", "flussonic-ts"] + .iter() + .any(|alias| mode.eq_ignore_ascii_case(alias)) + { + "fs" + } else if mode.eq_ignore_ascii_case("vod") { + "vod" + } else { + mode + } +} + +fn effective_catchup_mode(catchup: &CatchupProperties) -> &str { + catchup + .mode + .as_deref() + .map(str::trim) + .filter(|mode| !mode.is_empty()) + .or_else(|| catchup.catchup_type.as_deref().map(str::trim).filter(|mode| !mode.is_empty())) + .unwrap_or_default() +} + +fn parse_template(template: &str) -> Vec { + let mut segments = Vec::new(); + let mut current = String::new(); + let bytes = template.as_bytes(); + let mut idx = 0usize; + + while idx < bytes.len() { + let start = if bytes[idx] == b'{' { + Some((idx, 1usize)) + } else if idx + 1 < bytes.len() && bytes[idx] == b'$' && bytes[idx + 1] == b'{' { + Some((idx, 2usize)) + } else { + None + }; + + let Some((placeholder_start, open_len)) = start else { + current.push(bytes[idx] as char); + idx += 1; + continue; + }; + + if !current.is_empty() { + segments.push(TemplateSegment::Literal(std::mem::take(&mut current))); + } + + let mut end = placeholder_start + open_len; + while end < bytes.len() && bytes[end] != b'}' { + end += 1; + } + if end >= bytes.len() { + current.push_str(&template[placeholder_start..]); + break; + } + + segments.push(TemplateSegment::Placeholder(template[placeholder_start..=end].to_string())); + idx = end + 1; + } + + if !current.is_empty() { + segments.push(TemplateSegment::Literal(current)); + } + + segments +} + +fn collect_placeholders(segments: &[TemplateSegment]) -> Vec<&str> { + segments + .iter() + .filter_map(|segment| match segment { + TemplateSegment::Placeholder(value) => Some(value.as_str()), + TemplateSegment::Literal(_) => None, + }) + .collect() +} + +fn build_local_source( + base_url: &str, + path: &str, + token: &str, + placeholders: &[&str], + append_mode: bool, +) -> String { + let mut source = if append_mode { + format!("?{M3U_CATCHUP_MARKER}={token}") + } else { + format!("{base_url}/{M3U_CATCHUP_ROUTE_PREFIX}/{token}") + }; + if !append_mode && !path.is_empty() { + source.push('/'); + source.push_str(path.trim_start_matches('/')); + } + + for (idx, placeholder) in placeholders.iter().enumerate() { + let separator = if idx == 0 && !source.contains('?') { '?' } else { '&' }; + source.push(separator); + source.push_str(COLLECTOR_PREFIX); + source.push_str(&idx.to_string()); + source.push('='); + source.push_str(placeholder); + } + + source +} + +fn append_siptv_template(source_url: &str) -> String { + if source_url.contains('?') { + format!("&{}", M3U_APPEND_TEMPLATE.trim_start_matches('?')) + } else { + M3U_APPEND_TEMPLATE.to_string() + } +} + +fn derive_xc_template(source_url: &str) -> Option { + let parsed = Url::parse(source_url).ok()?; + let mut segments = parsed + .path_segments() + .map(|it| it.map(ToString::to_string).collect::>())?; + if segments.len() < 3 { + return None; + } + + let file_name = segments.pop()?; + let password = segments.pop()?; + let username = segments.pop()?; + if segments.last().is_some_and(|last| last == "live") { + let _ = segments.pop(); + } + segments.push("timeshift".to_string()); + segments.push(username); + segments.push(password); + segments.push("{duration:60}".to_string()); + segments.push(XC_START_TEMPLATE.to_string()); + segments.push(file_name); + + let mut rebuilt = parsed; + let new_path = format!("/{}", segments.join("/")); + rebuilt.set_path(&new_path); + Some(rebuilt.into()) +} + +fn derive_flussonic_template(source_url: &str) -> Option { + let mut parsed = Url::parse(source_url).ok()?; + let mut segments = parsed + .path_segments() + .map(|it| it.map(ToString::to_string).collect::>())?; + let file_name = segments.pop()?; + segments.push(format!( + "timeshift_abs-{FLUSSONIC_UTC_SENTINEL}{}", + extract_suffix_from_filename(&file_name) + )); + parsed.set_path(&format!("/{}", segments.join("/"))); + Some(String::from(parsed).replace(FLUSSONIC_UTC_SENTINEL, "{utc}")) +} + +fn append_query_template(source_url: &str, source_template: &str) -> Option { + let mut parsed = Url::parse(source_url).ok()?; + let append = source_template.strip_prefix('?').unwrap_or(source_template); + let mut merged = parsed.query().map(ToString::to_string).unwrap_or_default(); + if !merged.is_empty() && !append.is_empty() { + merged.push('&'); + } + merged.push_str(append); + parsed.set_query((!merged.is_empty()).then_some(merged.as_str())); + Some(parsed.into()) +} + +fn extract_suffix_from_filename(file_name: &str) -> &str { + file_name + .char_indices() + .find_map(|(idx, ch)| (ch == '.').then_some(&file_name[idx..])) + .unwrap_or_default() +} + +fn is_append_like_query_source(mode: &str, source: &str) -> bool { + mode_alias(mode) == "append" || source.starts_with('?') +} + +fn derived_template_for_mode<'a>(source_url: &'a str, catchup: &'a CatchupProperties) -> Option> { + let mode = effective_catchup_mode(catchup); + if let Some(source) = catchup.source.as_deref().filter(|source| !source.is_empty()) { + return Some(if is_append_like_query_source(mode, source) { + append_query_template(source_url, source).map(Cow::Owned)? + } else { + Cow::Borrowed(source) + }); + } + + match mode_alias(mode) { + "shift" => Some(Cow::Owned(format!("{source_url}{}", append_siptv_template(source_url)))), + "xc" => derive_xc_template(source_url).map(Cow::Owned), + "fs" => derive_flussonic_template(source_url).map(Cow::Owned), + "vod" => Some(Cow::Borrowed("{catchup-id}")), + _ => None, + } +} + +fn resolve_collectors(raw_query: Option<&str>, placeholders: &[&str]) -> Vec<(usize, String)> { + let mut collectors: Vec> = vec![None; placeholders.len()]; + let Some(raw_query) = raw_query else { + return into_collectors(collectors); + }; + + let params: Vec<(String, String)> = url::form_urlencoded::parse(raw_query.as_bytes()) + .filter(|(key, _)| !key.eq_ignore_ascii_case(M3U_CATCHUP_MARKER)) + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect(); + + // First pass: named mapping for native SIPTV-style placeholders. + // Smart IPTV strips the proxy marker and indexed `v0`/`v1` keys, leaving + // only `utc`/`lutc` in the request. Map those by placeholder name so the + // upstream URL still gets the correct values. + for (collector, placeholder) in collectors.iter_mut().zip(placeholders) { + let name = placeholder_name(placeholder); + if let Some((_, value)) = params.iter().find(|(key, _)| query_key_matches_placeholder(key, name)) { + *collector = Some(value.clone()); + } + } + + // Second pass: indexed mapping for proxy-rewritten requests carrying + // `v0`/`v1` collectors, falling back only where the named pass did not + // already fill the slot (so callers can mix both styles in one query). + for (key, value) in ¶ms { + let Some(idx) = key + .strip_prefix(COLLECTOR_PREFIX) + .and_then(|suffix| suffix.parse::().ok()) else { + continue; + }; + if let Some(collector) = collectors.get_mut(idx).filter(|collector| collector.is_none()) { + *collector = Some(value.clone()); + } + } + + fill_collectors_from_utc_lutc(&mut collectors, placeholders, ¶ms); + + into_collectors(collectors) +} + +fn query_key_matches_placeholder(key: &str, placeholder: &str) -> bool { + if key.eq_ignore_ascii_case(placeholder) { + return true; + } + if ["timestamp", "utcstart", "utc"] + .iter() + .any(|name| placeholder.eq_ignore_ascii_case(name)) + { + return key.eq_ignore_ascii_case("timestamp") + || key.eq_ignore_ascii_case("utcstart") + || key.eq_ignore_ascii_case(M3U_CATCHUP_PARAM_UTC); + } + false +} + +fn fill_collectors_from_utc_lutc( + collectors: &mut [Option], + placeholders: &[&str], + params: &[(String, String)], +) { + let utc = params + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(M3U_CATCHUP_PARAM_UTC)) + .map(|(_, value)| value.as_str()); + let duration = utc + .and_then(|start| start.parse::().ok()) + .zip( + params + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(M3U_CATCHUP_PARAM_LUTC)) + .and_then(|(_, value)| value.parse::().ok()), + ) + .and_then(|(start, end)| (end >= start).then(|| (end - start).to_string())); + + for (collector, placeholder) in collectors.iter_mut().zip(placeholders) { + if collector.is_some() { + continue; + } + let name = placeholder_name(placeholder); + if ["timestamp", "utcstart", "utc"].iter().any(|candidate| name.eq_ignore_ascii_case(candidate)) { + if let Some(value) = utc { + *collector = Some(value.to_string()); + } + } else if ["offset", "duration"].iter().any(|candidate| name.eq_ignore_ascii_case(candidate)) { + if let Some(value) = duration.as_ref() { + *collector = Some(value.clone()); + } + } + } +} + +fn into_collectors(collectors: Vec>) -> Vec<(usize, String)> { + collectors + .into_iter() + .enumerate() + .filter_map(|(idx, val)| val.map(|v| (idx, v))) + .collect() +} + +fn placeholder_name(placeholder: &str) -> &str { + let trimmed = placeholder.strip_prefix("${").unwrap_or(placeholder); + let trimmed = trimmed.strip_prefix('{').unwrap_or(trimmed); + trimmed.strip_suffix('}').unwrap_or(trimmed) +} + +fn render_template(segments: &[TemplateSegment], collectors: &[(usize, String)]) -> Result { + let mut collector_idx = 0usize; + let mut output = String::new(); + + for segment in segments { + match segment { + TemplateSegment::Literal(literal) => output.push_str(literal), + TemplateSegment::Placeholder(placeholder) => { + let Some((actual_idx, value)) = collectors.get(collector_idx) else { + return Err(TuliproxError::Crypto("Missing catchup collector value".to_string())); + }; + if *actual_idx != collector_idx { + return Err(TuliproxError::Crypto("Catchup collector sequence is sparse".to_string())); + } + let strips_sign = ["offset", "duration"] + .iter() + .any(|name| placeholder_name(placeholder).eq_ignore_ascii_case(name)); + let value = if strips_sign && output.ends_with('-') { + value.strip_prefix('-').unwrap_or(value) + } else if strips_sign && output.ends_with('+') { + value.strip_prefix('+').unwrap_or(value) + } else { + value + }; + output.push_str(value); + collector_idx += 1; + } + } + } + + if collector_idx != collectors.len() { + return Err(TuliproxError::Crypto("Unexpected extra catchup collector values".to_string())); + } + + Ok(output) +} + +pub fn build_m3u_catchup_rewrite( + secret: &[u8; 16], + base_url: &str, + username: &str, + target_id: u16, + virtual_id: u32, + source_url: &str, + catchup: &CatchupProperties, +) -> Result, TuliproxError> { + if catchup.native_flussonic_player_mode().is_some() { + return Ok(None); + } + let Some(template) = derived_template_for_mode(source_url, catchup) else { + return Ok(None); + }; + let token = encode_m3u_catchup_token( + secret, + &M3uCatchupToken { + username: username.to_string(), + target_id, + virtual_id, + }, + )?; + let segments = parse_template(template.as_ref()); + let placeholders = collect_placeholders(&segments); + let append_mode = catchup + .source + .as_deref() + .filter(|source| !source.is_empty()) + .is_some_and(|source| is_append_like_query_source(effective_catchup_mode(catchup), source)); + + let source = build_local_source(base_url, "", &token, &placeholders, append_mode); + let mode = if append_mode { "append" } else { "default" }; + Ok(Some(M3uCatchupRewrite { mode: mode.intern(), source: source.intern() })) +} + +pub fn has_m3u_catchup_marker(raw_query: Option<&str>) -> bool { + raw_query.is_some_and(|query| { + let mut explicit = false; + let mut wink_start = false; + let mut wink_window = false; + for (key, _) in url::form_urlencoded::parse(query.as_bytes()) { + explicit |= key.eq_ignore_ascii_case(M3U_CATCHUP_MARKER) + || key.eq_ignore_ascii_case(M3U_CATCHUP_PARAM_UTC) + || key.eq_ignore_ascii_case(M3U_CATCHUP_PARAM_LUTC); + wink_start |= key.eq_ignore_ascii_case("utcstart"); + wink_window |= key.eq_ignore_ascii_case("offset") || key.eq_ignore_ascii_case("duration"); + } + explicit || (wink_start && wink_window) + }) +} + +pub fn is_xtream_m3u_catchup_supported(source_url: &str, catchup: &CatchupProperties) -> bool { + xtream_m3u_catchup_segments(source_url, catchup).is_some() +} + +fn xtream_m3u_catchup_segments( + source_url: &str, + catchup: &CatchupProperties, +) -> Option> { + let template = derived_template_for_mode(source_url, catchup)?; + let segments = parse_template(template.as_ref()); + let mut has_placeholder = false; + for segment in &segments { + if let TemplateSegment::Placeholder(value) = segment { + has_placeholder = true; + if !matches!(value.as_str(), "{utc}" | "{duration}") { + return None; + } + } + } + has_placeholder.then_some(segments) +} + +fn parse_xtream_bridge_start(start: &str) -> Result { + if let Ok(ts) = start.parse::() { + return DateTime::from_timestamp(ts, 0).map(|_| ts).ok_or_else(|| { + TuliproxError::RepositoryM3u(format!("Invalid Xtream start timestamp: {start}")) + }); + } + for fmt in XTREAM_BRIDGE_START_FORMATS { + if let Ok(dt) = NaiveDateTime::parse_from_str(start, fmt) { + return Ok(dt.and_utc().timestamp()); + } + } + Err(TuliproxError::RepositoryM3u(format!( + "Unsupported Xtream start time format: {start}" + ))) +} + +pub fn resolve_xtream_m3u_catchup_url( + source_url: &str, + catchup: &CatchupProperties, + start: &str, + duration_minutes: &str, +) -> Result { + let segments = xtream_m3u_catchup_segments(source_url, catchup).ok_or_else(|| { + TuliproxError::RepositoryM3u( + "M3U catch-up template is not supported by the Xtream bridge".to_string(), + ) + })?; + let utc_start = parse_xtream_bridge_start(start)?; + let minutes: u64 = duration_minutes.parse().map_err(|_| { + TuliproxError::RepositoryM3u(format!("Invalid Xtream duration minutes: {duration_minutes}")) + })?; + let duration_secs = minutes.checked_mul(60).ok_or_else(|| { + TuliproxError::RepositoryM3u("Xtream duration minutes overflow seconds".to_string()) + })?; + + let collectors: Vec<(usize, String)> = segments + .iter() + .filter_map(|segment| match segment { + TemplateSegment::Placeholder(value) => Some(value.as_str()), + TemplateSegment::Literal(_) => None, + }) + .enumerate() + .map(|(idx, p)| { + let value = if p == "{utc}" { + utc_start.to_string() + } else { + duration_secs.to_string() + }; + (idx, value) + }) + .collect(); + + let url = render_template(&segments, &collectors)?; + let discriminator = short_hash(&url); + Ok(ResolvedM3uCatchup { url, discriminator }) +} + +pub fn resolve_m3u_catchup_url( + source_url: &str, + catchup: &CatchupProperties, + raw_query: Option<&str>, +) -> Result, TuliproxError> { + let Some(template) = derived_template_for_mode(source_url, catchup) else { + return Ok(None); + }; + let segments = parse_template(template.as_ref()); + let placeholders = collect_placeholders(&segments); + let collectors = resolve_collectors(raw_query, &placeholders); + if collectors.len() != placeholders.len() { + return Err(TuliproxError::Crypto(format!( + "Catchup collector mismatch: expected {}, got {}", + placeholders.len(), + collectors.len() + ))); + } + + let url = render_template(&segments, &collectors)?; + let mut discriminator = url::form_urlencoded::Serializer::new(String::new()); + let mode = effective_catchup_mode(catchup); + discriminator.append_pair("mode", if mode.is_empty() { "default" } else { mode }); + for (idx, value) in collectors { + discriminator.append_pair(&format!("{COLLECTOR_PREFIX}{idx}"), &value); + } + let discriminator = short_hash(&discriminator.finish()); + Ok(Some(ResolvedM3uCatchup { url, discriminator })) +} + +#[cfg(test)] +mod tests { + use super::{ + build_m3u_catchup_rewrite, has_m3u_catchup_marker, is_xtream_m3u_catchup_supported, + parse_template, render_template, resolve_m3u_catchup_url, resolve_xtream_m3u_catchup_url, + M3U_CATCHUP_MARKER, + }; + use shared::{error::TuliproxError, model::CatchupProperties, utils::Internable}; + + #[test] + fn explicit_append_rewrite_uses_live_route_marker() { + let rewrite = build_m3u_catchup_rewrite( + &[7u8; 16], + "http://proxy.example", + "alice", + 7, + 42, + "http://provider.example/live/42.m3u8", + &CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + ) + .unwrap() + .unwrap(); + + assert_eq!(rewrite.mode.as_ref(), "append"); + assert!(rewrite.source.starts_with("?tuliprox-catchup=")); + assert!(rewrite.source.contains("&v0=${offset}")); + assert!(rewrite.source.contains("&v1=${timestamp}")); + } + + #[test] + fn shift_rewrite_uses_direct_catchup_route() { + let rewrite = build_m3u_catchup_rewrite( + &[7u8; 16], + "http://proxy.example", + "alice", + 7, + 42, + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("shift".intern()), + ..CatchupProperties::default() + }, + ) + .unwrap() + .unwrap(); + + assert_eq!(rewrite.mode.as_ref(), "default"); + assert!(rewrite.source.starts_with("http://proxy.example/m3u-catchup/")); + assert!(rewrite.source.contains("?v0={utc}")); + assert!(rewrite.source.contains("&v1={lutc}")); + } + + #[test] + fn resolve_explicit_append_template_roundtrips_collectors() { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.m3u8", + &CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + Some("tuliprox-catchup=abc&v0=120&v1=1717200000"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.m3u8?offset=-120&utcstart=1717200000" + ); + assert!(!resolved.discriminator.is_empty()); + } + + #[test] + fn resolve_explicit_append_template_merges_into_existing_query() { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.m3u8?token=abc", + &CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + Some("tuliprox-catchup=abc&v0=120&v1=1717200000"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.m3u8?token=abc&offset=-120&utcstart=1717200000" + ); + } + + #[test] + fn resolve_default_mode_query_template_keeps_base_stream_url() { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("default".intern()), + source: Some("?playseek=${timestamp}&duration=${duration}".intern()), + ..CatchupProperties::default() + }, + Some("v0=1717200000&v1=120"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.ts?playseek=1717200000&duration=120" + ); + } + + #[test] + fn default_mode_query_template_rewrite_uses_append_route_marker() { + let rewrite = build_m3u_catchup_rewrite( + &[7u8; 16], + "http://proxy.example", + "alice", + 7, + 42, + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("default".intern()), + source: Some("?playseek=${timestamp}&duration=${duration}".intern()), + ..CatchupProperties::default() + }, + ) + .unwrap() + .unwrap(); + + assert_eq!(rewrite.mode.as_ref(), "append"); + assert!(rewrite.source.starts_with("?tuliprox-catchup=")); + assert!(rewrite.source.contains("&v0=${timestamp}")); + assert!(rewrite.source.contains("&v1=${duration}")); + } + + #[test] + fn resolve_shift_template_roundtrips_collectors() { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("shift".intern()), + ..CatchupProperties::default() + }, + Some("v0=20240101120000&v1=20240101130000"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.ts?utc=20240101120000&lutc=20240101130000" + ); + } + + #[test] + fn discriminator_changes_when_collectors_change() { + let first = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("shift".intern()), + ..CatchupProperties::default() + }, + Some("v0=20240101120000&v1=20240101130000"), + ) + .unwrap() + .unwrap(); + let second = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("shift".intern()), + ..CatchupProperties::default() + }, + Some("v0=20240101140000&v1=20240101150000"), + ) + .unwrap() + .unwrap(); + + assert_ne!(first.discriminator, second.discriminator); + } + + #[test] + fn catchup_marker_detection_is_explicit() { + assert!(has_m3u_catchup_marker(Some(&format!("{M3U_CATCHUP_MARKER}=abc&v0=1")))); + assert!(has_m3u_catchup_marker(Some("TULIPROX-CATCHUP=abc"))); + assert!(has_m3u_catchup_marker(Some("UTC=1784869500"))); + assert!(has_m3u_catchup_marker(Some("LUTC=1784877418"))); + assert!(!has_m3u_catchup_marker(Some("v0=1"))); + assert!(!has_m3u_catchup_marker(None)); + } + + #[test] + fn render_template_only_strips_signs_from_offset_and_duration() -> Result<(), TuliproxError> { + let segments = parse_template("timeshift_abs-{utc}-{duration}"); + let collectors = vec![(0, "-1784885700".to_string()), (1, "-120".to_string())]; + + assert_eq!(render_template(&segments, &collectors)?, "timeshift_abs--1784885700-120"); + Ok(()) + } + + #[test] + fn siptv_style_catchup_request_is_detected() { + // Smart IPTV keeps `utc`/`lutc` and strips the proxy marker. + assert!(has_m3u_catchup_marker(Some("utc=1784869500&lutc=1784877418"))); + assert!(has_m3u_catchup_marker(Some("lutc=1784877418"))); + assert!(!has_m3u_catchup_marker(Some("foo=1"))); + } + + #[test] + fn wink_catchup_detection_requires_start_and_window() { + assert!(has_m3u_catchup_marker(Some("utcstart=1717200000&offset=-120"))); + assert!(has_m3u_catchup_marker(Some("duration=120&utcstart=1717200000"))); + assert!(!has_m3u_catchup_marker(Some("utcstart=1717200000"))); + assert!(!has_m3u_catchup_marker(Some("start=1717200000&duration=120"))); + assert!(!has_m3u_catchup_marker(Some("timestamp=1717200000"))); + } + + #[test] + fn resolve_append_template_from_wink_named_collectors() -> Result<(), TuliproxError> { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/hls/channel/index.m3u8?useseq=t", + &CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + Some("offset=-120&utcstart=1717200000"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u("Wink catch-up URL was not resolved".to_string()))?; + + assert_eq!( + resolved.url, + "http://provider.example/hls/channel/index.m3u8?useseq=t&offset=-120&utcstart=1717200000" + ); + Ok(()) + } + + #[test] + fn wink_offset_sign_is_preserved_when_template_has_no_literal_minus() -> Result<(), TuliproxError> { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + Some("offset=-120&utcstart=1717200000"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u("signed Wink URL was not resolved".to_string()))?; + + assert_eq!(resolved.url, "http://provider.example/live/42.ts?offset=-120&utcstart=1717200000"); + Ok(()) + } + + #[test] + fn resolve_append_template_derives_duration_from_utc_lutc() -> Result<(), TuliproxError> { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + catchup_type: Some("append".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + Some("utc=1717200000&lutc=1717200120"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u("TiviMate append URL was not resolved".to_string()))?; + + assert_eq!(resolved.url, "http://provider.example/live/42.ts?offset=-120&utcstart=1717200000"); + Ok(()) + } + + #[test] + fn resolve_shift_template_from_siptv_native_collectors() { + // Smart IPTV strips `tuliprox-catchup`/`v0`/`v1` and forwards only + // `utc`/`lutc`. The server must still resolve the upstream URL. + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("shift".intern()), + ..CatchupProperties::default() + }, + Some("utc=20240101120000&lutc=20240101130000"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.ts?utc=20240101120000&lutc=20240101130000" + ); + } + + #[test] + fn resolve_flussonic_hls_template_replaces_utc() -> Result<(), TuliproxError> { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/channel/mono.m3u8?token=abc", + &CatchupProperties { + mode: Some("fs".intern()), + ..CatchupProperties::default() + }, + Some("utc=1784885700&lutc=1784899708"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u("HLS Flussonic URL was not resolved".to_string()))?; + + assert_eq!( + resolved.url, + "http://provider.example/channel/timeshift_abs-1784885700.m3u8?token=abc" + ); + assert!(!resolved.url.contains("%7B")); + Ok(()) + } + + #[test] + fn resolve_flussonic_ts_template_replaces_utc() -> Result<(), TuliproxError> { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/channel/channel.ts?token=abc", + &CatchupProperties { + mode: Some("fs".intern()), + ..CatchupProperties::default() + }, + Some("utc=1784885700&lutc=1784899708"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u("TS Flussonic URL was not resolved".to_string()))?; + + assert_eq!( + resolved.url, + "http://provider.example/channel/timeshift_abs-1784885700.ts?token=abc" + ); + assert!(!resolved.url.contains("%7B")); + Ok(()) + } + + #[test] + fn flussonic_aliases_are_case_insensitive() -> Result<(), TuliproxError> { + for mode in ["fs", "FLUSSONIC", "Flussonic-Hls", "flussonic-TS"] { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/channel/mono.m3u8", + &CatchupProperties { + mode: Some(mode.intern()), + ..CatchupProperties::default() + }, + Some("utc=1784885700"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u(format!("mode {mode} was not resolved")))?; + + assert_eq!( + resolved.url, + "http://provider.example/channel/timeshift_abs-1784885700.m3u8" + ); + } + Ok(()) + } + + #[test] + fn catchup_type_flussonic_is_used_when_mode_is_missing() -> Result<(), TuliproxError> { + let resolved = resolve_m3u_catchup_url( + "http://provider.example/channel/channel.ts", + &CatchupProperties { + catchup_type: Some("flussonic".intern()), + ..CatchupProperties::default() + }, + Some("utc=1784885700"), + )? + .ok_or_else(|| TuliproxError::RepositoryM3u("catchup-type was not resolved".to_string()))?; + + assert_eq!( + resolved.url, + "http://provider.example/channel/timeshift_abs-1784885700.ts" + ); + Ok(()) + } + + #[test] + fn native_flussonic_rewrite_uses_player_path_instead_of_token_source() -> Result<(), TuliproxError> { + let rewrite = build_m3u_catchup_rewrite( + &[7u8; 16], + "http://proxy.example", + "alice", + 7, + 42, + "http://provider.example/ch/index.m3u8", + &CatchupProperties { + catchup_type: Some("flussonic".intern()), + days: Some("3".intern()), + ..CatchupProperties::default() + }, + )?; + + assert!(rewrite.is_none()); + Ok(()) + } + + #[test] + fn resolve_append_template_keeps_indexed_collectors() { + // Plain indexed collectors without the marker must still work; this + // is the path used by Smart STB / non-stripping clients. + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.m3u8", + &CatchupProperties { + mode: Some("append".intern()), + source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), + ..CatchupProperties::default() + }, + Some("v0=120&v1=1717200000"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.m3u8?offset=-120&utcstart=1717200000" + ); + } + + #[test] + fn resolve_collectors_mix_named_and_indexed() { + // A request that carries both styles should fill the template + // placeholders without overlap. + let resolved = resolve_m3u_catchup_url( + "http://provider.example/live/42.ts", + &CatchupProperties { + mode: Some("shift".intern()), + ..CatchupProperties::default() + }, + Some("utc=20240101120000&v1=20240101130000"), + ) + .unwrap() + .unwrap(); + + assert_eq!( + resolved.url, + "http://provider.example/live/42.ts?utc=20240101120000&lutc=20240101130000" + ); + } + + #[test] + fn xtream_bridge_resolves_flussonic_utc_and_duration() { + let catchup = CatchupProperties { + mode: Some("flussonic".intern()), + source: Some("http://provider.example/channel/video-{utc}-{duration}.m3u8".intern()), + ..CatchupProperties::default() + }; + let resolved = resolve_xtream_m3u_catchup_url( + "http://provider.example/channel/index.m3u8", + &catchup, + "2024-01-01:00-00", + "60", + ) + .expect("valid Flussonic template"); + assert_eq!( + resolved.url, + "http://provider.example/channel/video-1704067200-3600.m3u8" + ); + assert!(!resolved.discriminator.is_empty()); + } + + #[test] + fn xtream_bridge_rejects_unsupported_placeholder() { + let catchup = CatchupProperties { + mode: Some("flussonic".intern()), + source: Some("http://provider.example/channel/${timestamp}.m3u8".intern()), + ..CatchupProperties::default() + }; + assert!(!is_xtream_m3u_catchup_supported( + "http://provider.example/channel/index.m3u8", + &catchup + )); + let err = resolve_xtream_m3u_catchup_url( + "http://provider.example/channel/index.m3u8", + &catchup, + "2024-01-01:00-00", + "60", + ) + .unwrap_err(); + assert!(matches!(err, TuliproxError::RepositoryM3u(_))); + } + + #[test] + fn xtream_bridge_rejects_invalid_start_time() { + let catchup = CatchupProperties { + mode: Some("flussonic".intern()), + source: Some("http://provider.example/channel/video-{utc}-{duration}.m3u8".intern()), + ..CatchupProperties::default() + }; + let err = resolve_xtream_m3u_catchup_url( + "http://provider.example/channel/index.m3u8", + &catchup, + "not-a-timestamp", + "60", + ) + .unwrap_err(); + assert!(matches!(err, TuliproxError::RepositoryM3u(_))); + } + + #[test] + fn xtream_bridge_rejects_overflowing_duration_minutes() { + let catchup = CatchupProperties { + mode: Some("flussonic".intern()), + source: Some("http://provider.example/channel/video-{utc}-{duration}.m3u8".intern()), + ..CatchupProperties::default() + }; + let err = resolve_xtream_m3u_catchup_url( + "http://provider.example/channel/index.m3u8", + &catchup, + "2024-01-01:00-00", + &u64::MAX.to_string(), + ) + .unwrap_err(); + assert!(matches!(err, TuliproxError::RepositoryM3u(_))); + } + + #[test] + fn xtream_bridge_accepts_unix_seconds_start() { + let catchup = CatchupProperties { + mode: Some("flussonic".intern()), + source: Some("http://provider.example/channel/video-{utc}-{duration}.m3u8".intern()), + ..CatchupProperties::default() + }; + let resolved = resolve_xtream_m3u_catchup_url( + "http://provider.example/channel/index.m3u8", + &catchup, + "1704067200", + "60", + ) + .expect("unix seconds start"); + assert_eq!( + resolved.url, + "http://provider.example/channel/video-1704067200-3600.m3u8" + ); + } +} diff --git a/backend/src/utils/m3u_catchup_token.rs b/backend/src/iptv/m3u/catchup_token.rs similarity index 100% rename from backend/src/utils/m3u_catchup_token.rs rename to backend/src/iptv/m3u/catchup_token.rs diff --git a/backend/src/utils/network/m3u.rs b/backend/src/iptv/m3u/client.rs similarity index 100% rename from backend/src/utils/network/m3u.rs rename to backend/src/iptv/m3u/client.rs diff --git a/backend/src/iptv/m3u/mod.rs b/backend/src/iptv/m3u/mod.rs new file mode 100644 index 000000000..d1d2da87e --- /dev/null +++ b/backend/src/iptv/m3u/mod.rs @@ -0,0 +1,9 @@ +mod archive; +mod catchup; +mod catchup_token; +pub mod client; + +pub use archive::*; +pub use catchup::*; +pub use catchup_token::*; +pub use client::*; diff --git a/backend/src/iptv/mod.rs b/backend/src/iptv/mod.rs new file mode 100644 index 000000000..96a02c094 --- /dev/null +++ b/backend/src/iptv/mod.rs @@ -0,0 +1,3 @@ +pub mod m3u; +pub mod stalker; +pub mod xtream; diff --git a/backend/src/utils/network/stalker/auth.rs b/backend/src/iptv/stalker/auth.rs similarity index 91% rename from backend/src/utils/network/stalker/auth.rs rename to backend/src/iptv/stalker/auth.rs index 0ce81a12a..e6ec20f4c 100644 --- a/backend/src/utils/network/stalker/auth.rs +++ b/backend/src/iptv/stalker/auth.rs @@ -3,15 +3,15 @@ use serde::Deserialize; use serde_json::Value; use shared::model::stalker::{StalkerAuthMode, StalkerBootstrapRecipe, StalkerPortalCapabilitiesDto}; -use crate::utils::network::stalker::client::StalkerApiClient; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; -use crate::utils::network::stalker::presets::stalker_mag_preset_spec; -use crate::utils::network::stalker::profile::{ +use crate::iptv::stalker::client::StalkerApiClient; +use crate::iptv::stalker::error::{safe_stalker_url, StalkerError, StalkerResult}; +use crate::iptv::stalker::presets::stalker_mag_preset_spec; +use crate::iptv::stalker::profile::{ StalkerHandshake, StalkerProviderProfile, StalkerRawProviderProfile, }; -use crate::utils::network::stalker::recipes::{detect_fingerprint, fallback_recipes_for, recipe_spec_for}; -use crate::utils::network::stalker::session::StalkerSession; -use crate::utils::network::stalker::url_factory::StalkerLoadUrl; +use crate::iptv::stalker::recipes::{detect_fingerprint, fallback_recipes_for, recipe_spec_for}; +use crate::iptv::stalker::session::StalkerSession; +use crate::iptv::stalker::url_factory::StalkerLoadUrl; /// The fields we expect in a successful handshake response. The portal wraps the result /// in `{"js": {...}}` — we accept both wrapped and unwrapped shapes for robustness. @@ -66,7 +66,7 @@ pub async fn handshake(client: &StalkerApiClient) -> StalkerResult = None; for load_url in candidates { @@ -113,7 +113,10 @@ async fn attempt_recipe(client: &StalkerApiClient, recipe: StalkerBootstrapRecip }; let fingerprint = detect_fingerprint(handshake_status, &session.fingerprint_evidence, client.config().mag_preset); - debug!("Stalker portal fingerprint for {}: {fingerprint:?}", load_url.load_url); + debug!( + "Stalker portal fingerprint for {}: {fingerprint:?}", + safe_stalker_url(&load_url.load_url) + ); let capabilities = fetch_capabilities(client, &session, &load_url, &spec).await.unwrap_or_default(); let size_caps = client .config() @@ -145,7 +148,7 @@ async fn attempt_recipe(client: &StalkerApiClient, recipe: StalkerBootstrapRecip async fn perform_handshake_against( client: &StalkerApiClient, load_url: &StalkerLoadUrl, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, ) -> StalkerResult<(StalkerSession, u16)> { let config = client.config(); let preset_spec = stalker_mag_preset_spec(config.mag_preset); @@ -171,12 +174,12 @@ async fn perform_handshake_against( if matches!(status.as_u16(), 401 | 403 | 456) { return Err(StalkerError::TokenRejected { status: status.as_u16(), - url: load_url.load_url.parse().ok(), + url: Some(load_url.load_url.as_str().into()), }); } return Err(StalkerError::HandshakeFailed { message: format!("handshake status {}", status.as_u16()), - url: load_url.load_url.parse().ok(), + url: Some(load_url.load_url.as_str().into()), }); } let parsed: StalkerHandshakeResponse = client.decode_body_bytes(&body, "handshake")?; @@ -194,7 +197,7 @@ async fn perform_handshake_against( let Some(token) = token else { return Err(StalkerError::HandshakeFailed { message: "no token in handshake response".to_string(), - url: load_url.load_url.parse().ok(), + url: Some(load_url.load_url.as_str().into()), }); }; let evidence = parsed @@ -240,7 +243,7 @@ async fn perform_do_auth( client: &StalkerApiClient, session: &StalkerSession, load_url: &StalkerLoadUrl, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, login: &str, password: &str, ) -> StalkerResult<()> { @@ -263,7 +266,7 @@ async fn perform_do_auth( if matches!(value.get("js"), Some(Value::Bool(false))) { return Err(StalkerError::HandshakeFailed { message: "portal rejected do_auth credentials".to_string(), - url: load_url.load_url.parse().ok(), + url: Some(load_url.load_url.as_str().into()), }); } Ok(()) @@ -273,7 +276,7 @@ async fn perform_handshake_extra( client: &StalkerApiClient, session: &mut StalkerSession, load_url: &StalkerLoadUrl, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, ) -> StalkerResult<()> { let mut builder = client .http() @@ -300,7 +303,7 @@ async fn perform_portal_handshake( client: &StalkerApiClient, session: &StalkerSession, load_url: &StalkerLoadUrl, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, ) -> StalkerResult<()> { let target = load_url; let mut builder = client @@ -328,7 +331,7 @@ async fn fetch_profile( client: &StalkerApiClient, session: &StalkerSession, load_url: &StalkerLoadUrl, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, ) -> StalkerResult { let mut builder = client .http() @@ -364,7 +367,7 @@ async fn fetch_capabilities( client: &StalkerApiClient, session: &StalkerSession, load_url: &StalkerLoadUrl, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, ) -> StalkerResult { let mut builder = client .http() diff --git a/backend/src/utils/network/stalker/catalog.rs b/backend/src/iptv/stalker/catalog.rs similarity index 98% rename from backend/src/utils/network/stalker/catalog.rs rename to backend/src/iptv/stalker/catalog.rs index a3c209afa..aafe04bbe 100644 --- a/backend/src/utils/network/stalker/catalog.rs +++ b/backend/src/iptv/stalker/catalog.rs @@ -4,11 +4,11 @@ use serde_json::Value; use shared::model::stalker::StalkerStreamKind; use shared::utils::{deserialize_as_option_string, deserialize_number_from_string}; -use crate::utils::network::stalker::client::StalkerApiClient; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; -use crate::utils::network::stalker::profile::StalkerHandshake; -use crate::utils::network::stalker::recipes::recipe_spec_for; -use crate::utils::network::stalker::url_factory::StalkerLoadUrl; +use crate::iptv::stalker::client::StalkerApiClient; +use crate::iptv::stalker::error::{safe_stalker_url, StalkerError, StalkerResult}; +use crate::iptv::stalker::profile::StalkerHandshake; +use crate::iptv::stalker::recipes::recipe_spec_for; +use crate::iptv::stalker::url_factory::StalkerLoadUrl; /// A category returned by `get_*_categories`. Stalker portals wrap the list in `{"js": [...]}`. #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] @@ -316,7 +316,7 @@ async fn get_categories( } } } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } fn parse_categories(value: &Value) -> Vec { @@ -571,7 +571,7 @@ async fn get_catalog_value( Err(err) => last_err = Some(err), } } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } pub async fn get_all_channels( @@ -674,7 +674,7 @@ pub async fn get_series_details( } } } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } async fn fetch_series_page( diff --git a/backend/src/utils/network/stalker/client.rs b/backend/src/iptv/stalker/client.rs similarity index 97% rename from backend/src/utils/network/stalker/client.rs rename to backend/src/iptv/stalker/client.rs index 311ca6ba7..bd2ada90b 100644 --- a/backend/src/utils/network/stalker/client.rs +++ b/backend/src/iptv/stalker/client.rs @@ -12,17 +12,17 @@ use serde::Deserialize; use tokio::sync::Mutex as AsyncMutex; use crate::model::{StalkerInputConfig, StalkerSizeCaps}; -use crate::utils::network::stalker::auth; -use crate::utils::network::stalker::catalog; -use crate::utils::network::stalker::cookie_jar::{apply_set_cookie_headers_unchecked, StalkerCookieJar}; -use crate::utils::network::stalker::epg; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; -use crate::utils::network::stalker::playback; -use crate::utils::network::stalker::presets::stalker_mag_preset_spec; -use crate::utils::network::stalker::profile::{StalkerHandshake, StalkerResolvedStream}; -use crate::utils::network::stalker::recipes::apply_endpoint_preference; -use crate::utils::network::stalker::session::StalkerSession; -use crate::utils::network::stalker::url_factory::{load_url_candidates, StalkerLoadUrl}; +use crate::iptv::stalker::auth; +use crate::iptv::stalker::catalog; +use crate::iptv::stalker::cookie_jar::{apply_set_cookie_headers_unchecked, StalkerCookieJar}; +use crate::iptv::stalker::epg; +use crate::iptv::stalker::error::{StalkerError, StalkerResult}; +use crate::iptv::stalker::playback; +use crate::iptv::stalker::presets::stalker_mag_preset_spec; +use crate::iptv::stalker::profile::{StalkerHandshake, StalkerResolvedStream}; +use crate::iptv::stalker::recipes::apply_endpoint_preference; +use crate::iptv::stalker::session::StalkerSession; +use crate::iptv::stalker::url_factory::{load_url_candidates, StalkerLoadUrl}; /// Stalker portals expect a `X-User-Agent` header on every call. The `reqwest` re-export /// does not ship a constant for it (the upstream convention is to use a custom name), so @@ -135,7 +135,7 @@ impl StalkerApiClient { // Honour the soft TTL: a cached session older than `STALKER_SESSION_TTL` // is discarded and re-handshaken. The portal may invalidate tokens // earlier; that path is caught by the 4xx retry hook in `api_utils.rs`. - if !active.session.is_stale(crate::utils::network::stalker::session::STALKER_SESSION_TTL) { + if !active.session.is_stale(crate::iptv::stalker::session::STALKER_SESSION_TTL) { return Ok(active); } // Stale: drop the cached handshake so the call below re-issues it. @@ -332,7 +332,7 @@ impl StalkerApiClient { } Err(err) => warn!("Stalker: dropping invalid Referer header value: {err}"), } - let now = crate::utils::network::stalker::cookie_jar::now_epoch_secs(); + let now = crate::iptv::stalker::cookie_jar::now_epoch_secs(); let mut cookie_pairs = identity_cookie_pairs(&self.config); for (name, value) in self.cookies.active_cookies(now) { // Server-set cookies win over our synthesized identity values. diff --git a/backend/src/utils/network/stalker/cmd_parser.rs b/backend/src/iptv/stalker/cmd_parser.rs similarity index 98% rename from backend/src/utils/network/stalker/cmd_parser.rs rename to backend/src/iptv/stalker/cmd_parser.rs index 46f2fbd10..8998d2952 100644 --- a/backend/src/utils/network/stalker/cmd_parser.rs +++ b/backend/src/iptv/stalker/cmd_parser.rs @@ -1,7 +1,7 @@ use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use url::Url; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; +use crate::iptv::stalker::error::{StalkerError, StalkerResult}; /// A Stalker `cmd` field is usually a plain-text space-prefixed command + URL pair, e.g. /// `"ffmpeg http://portal.example/stream/123"` or a bare URL. Some portals embed a diff --git a/backend/src/utils/network/stalker/cookie_jar.rs b/backend/src/iptv/stalker/cookie_jar.rs similarity index 99% rename from backend/src/utils/network/stalker/cookie_jar.rs rename to backend/src/iptv/stalker/cookie_jar.rs index 86f62fc79..9514f1879 100644 --- a/backend/src/utils/network/stalker/cookie_jar.rs +++ b/backend/src/iptv/stalker/cookie_jar.rs @@ -4,7 +4,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use parking_lot::RwLock; -use crate::utils::network::stalker::error::StalkerResult; +use crate::iptv::stalker::error::StalkerResult; /// The cookie jar is intentionally cheaply cloneable (the inner map is wrapped in a /// `parking_lot::RwLock` so cloning a jar shares the storage; this is safe because all diff --git a/backend/src/utils/network/stalker/epg.rs b/backend/src/iptv/stalker/epg.rs similarity index 98% rename from backend/src/utils/network/stalker/epg.rs rename to backend/src/iptv/stalker/epg.rs index e4764e056..16c7b0946 100644 --- a/backend/src/utils/network/stalker/epg.rs +++ b/backend/src/iptv/stalker/epg.rs @@ -9,10 +9,10 @@ use std::io::{Error as IoError, ErrorKind, Read}; use std::sync::{atomic::{AtomicBool, Ordering}, Arc}; use tokio_util::io::{StreamReader, SyncIoBridge}; -use crate::utils::network::stalker::client::StalkerApiClient; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; -use crate::utils::network::stalker::profile::StalkerHandshake; -use crate::utils::network::stalker::recipes::recipe_spec_for; +use crate::iptv::stalker::client::StalkerApiClient; +use crate::iptv::stalker::error::{safe_stalker_url, StalkerError, StalkerResult}; +use crate::iptv::stalker::profile::StalkerHandshake; +use crate::iptv::stalker::recipes::recipe_spec_for; /// A single EPG programme record. The portal wraps each entry in `{ ch_id, title, start, /// stop, ... }`. We accept any payload shape by deserialising into a permissive value @@ -120,7 +120,7 @@ pub async fn get_short_epg( } } } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } /// Per-channel full EPG. The portal returns the same payload shape as the short EPG. @@ -155,7 +155,7 @@ pub async fn get_epg( } } } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } /// Bulk-EPG stream. The portal can return hundreds of thousands of records in a single @@ -270,7 +270,7 @@ where } return Ok(()); } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } struct CancellableReader { diff --git a/backend/src/utils/network/stalker/error.rs b/backend/src/iptv/stalker/error.rs similarity index 72% rename from backend/src/utils/network/stalker/error.rs rename to backend/src/iptv/stalker/error.rs index 595ffab53..ff1cce72e 100644 --- a/backend/src/utils/network/stalker/error.rs +++ b/backend/src/iptv/stalker/error.rs @@ -2,6 +2,32 @@ use std::io; use thiserror::Error; use url::Url; +pub(crate) fn safe_stalker_url(value: &str) -> String { + let Ok(mut url) = Url::parse(value) else { + return "[redacted invalid URL]".to_string(); + }; + if !matches!(url.scheme(), "http" | "https") + || url.set_username("").is_err() + || url.set_password(None).is_err() + { + return "[redacted invalid URL]".to_string(); + } + url.set_query(None); + url.set_fragment(None); + url.into() +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StalkerErrorUrl(String); + +impl From<&str> for StalkerErrorUrl { + fn from(value: &str) -> Self { Self(safe_stalker_url(value)) } +} + +impl From for StalkerErrorUrl { + fn from(value: String) -> Self { Self::from(value.as_str()) } +} + /// Failure modes surfaced by the Stalker portal client. The variants cover both transport /// errors (network, body caps, JSONP decoding) and protocol-level errors (token rejection, /// portal refusal, missing playable URL). `Status`-flavored variants carry the upstream @@ -9,10 +35,10 @@ use url::Url; #[derive(Debug, Error)] pub enum StalkerError { #[error("stalker portal handshake failed: {message}")] - HandshakeFailed { message: String, url: Option }, + HandshakeFailed { message: String, url: Option }, #[error("stalker portal rejected the token (status {status})")] - TokenRejected { status: u16, url: Option }, + TokenRejected { status: u16, url: Option }, #[error("stalker portal body reported code {code} for {action}: {body_snippet}")] PortalBodyError { code: u16, action: String, body_snippet: String }, @@ -111,4 +137,21 @@ mod tests { } .is_token_rejected()); } + + #[test] + fn stalker_error_urls_drop_userinfo_query_and_fragment() { + let raw = "https://user:pass@portal.example/server/load.php?token=secret&mac=00:11:22:33:44:55#fragment"; + let errors = [ + StalkerError::HandshakeFailed { message: "failed".to_string(), url: Some(raw.into()) }, + StalkerError::TokenRejected { status: 401, url: Some(raw.into()) }, + ]; + + for error in errors { + let debug = format!("{error:?}"); + assert!(debug.contains("https://portal.example/server/load.php")); + for secret in ["user", "pass", "secret", "00:11:22:33:44:55", "fragment"] { + assert!(!debug.contains(secret), "error debug output leaked {secret}"); + } + } + } } diff --git a/backend/src/utils/network/stalker/mod.rs b/backend/src/iptv/stalker/mod.rs similarity index 100% rename from backend/src/utils/network/stalker/mod.rs rename to backend/src/iptv/stalker/mod.rs diff --git a/backend/src/utils/network/stalker/playback.rs b/backend/src/iptv/stalker/playback.rs similarity index 93% rename from backend/src/utils/network/stalker/playback.rs rename to backend/src/iptv/stalker/playback.rs index a3eec702d..ca358858b 100644 --- a/backend/src/utils/network/stalker/playback.rs +++ b/backend/src/iptv/stalker/playback.rs @@ -4,11 +4,11 @@ use serde_json::Value; use shared::model::stalker::{StalkerPlaybackMode, StalkerStreamKind}; use url::Url; -use crate::utils::network::stalker::client::{validate_playable_scheme, StalkerApiClient}; -use crate::utils::network::stalker::cmd_parser::scheme_is_playable; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; -use crate::utils::network::stalker::profile::{StalkerHandshake, StalkerResolvedStream}; -use crate::utils::network::stalker::recipes::recipe_spec_for; +use crate::iptv::stalker::client::{validate_playable_scheme, StalkerApiClient}; +use crate::iptv::stalker::cmd_parser::scheme_is_playable; +use crate::iptv::stalker::error::{safe_stalker_url, StalkerError, StalkerResult}; +use crate::iptv::stalker::profile::{StalkerHandshake, StalkerResolvedStream}; +use crate::iptv::stalker::recipes::recipe_spec_for; #[derive(Debug, Default, Clone, Deserialize, Serialize)] struct StalkerCreateLinkResponse { @@ -25,7 +25,7 @@ struct StalkerCreateLinkResponse { /// Resolve a `cmd` to a playable URL. The portal is told which kind of stream we want /// (`live`, `movie`, `episode` or `archive`) and the original `cmd` field. The response is /// expected to be a JSON object whose `js.cmd` field is a base64-encoded `ffmpeg ` -/// pair, with the URL extracted by [`crate::utils::network::stalker::cmd_parser`]. +/// pair, with the URL extracted by [`crate::iptv::stalker::cmd_parser`]. /// /// `requested_mode` is the playback mode the caller already decided on (from the /// item's temp-link capability flags). The resolved stream preserves this mode so @@ -56,15 +56,15 @@ pub async fn create_link( } } } - Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: client.portal_url().to_string() })) + Err(last_err.unwrap_or_else(|| StalkerError::NoEndpoint { portal: safe_stalker_url(client.portal_url()) })) } #[allow(clippy::too_many_arguments)] fn build_create_link_builder( client: &StalkerApiClient, - load_url: &crate::utils::network::stalker::url_factory::StalkerLoadUrl, + load_url: &crate::iptv::stalker::url_factory::StalkerLoadUrl, handshake: &StalkerHandshake, - spec: &crate::utils::network::stalker::recipes::StalkerRecipeSpec, + spec: &crate::iptv::stalker::recipes::StalkerRecipeSpec, kind: StalkerStreamKind, cmd: &str, series_number: Option, @@ -133,7 +133,7 @@ fn resolve_response( reason: "create_link response contained no cmd".to_string(), }); }; - let url = crate::utils::network::stalker::cmd_parser::extract_url_from_cmd(&raw_cmd)?; + let url = crate::iptv::stalker::cmd_parser::extract_url_from_cmd(&raw_cmd)?; let url = repair_live_stream_target(url, source_cmd, kind); let scheme = validate_playable_scheme(&url)?; if !scheme_is_playable(scheme) { @@ -167,7 +167,7 @@ fn repair_live_stream_target(url: String, source_cmd: &str, kind: StalkerStreamK } fn source_live_stream_target(source_cmd: &str) -> Option { - let source_url = crate::utils::network::stalker::cmd_parser::extract_url_from_cmd(source_cmd).ok()?; + let source_url = crate::iptv::stalker::cmd_parser::extract_url_from_cmd(source_cmd).ok()?; let parsed = Url::parse(&source_url).ok()?; if let Some(target) = parsed .query_pairs() @@ -231,7 +231,7 @@ fn upsert_raw_query_parameter(url: &str, name: &str, value: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::utils::network::stalker::client::strip_jsonp; + use crate::iptv::stalker::client::strip_jsonp; #[test] fn resolves_create_link_response_with_js_cmd() { diff --git a/backend/src/utils/network/stalker/presets.rs b/backend/src/iptv/stalker/presets.rs similarity index 100% rename from backend/src/utils/network/stalker/presets.rs rename to backend/src/iptv/stalker/presets.rs diff --git a/backend/src/utils/network/stalker/profile.rs b/backend/src/iptv/stalker/profile.rs similarity index 99% rename from backend/src/utils/network/stalker/profile.rs rename to backend/src/iptv/stalker/profile.rs index d80c8f97e..f5f7bc513 100644 --- a/backend/src/utils/network/stalker/profile.rs +++ b/backend/src/iptv/stalker/profile.rs @@ -8,7 +8,7 @@ use shared::model::stalker::{ use shared::utils::deserialize_as_option_string; use crate::model::{StalkerInputConfig, StalkerSizeCaps}; -use crate::utils::network::stalker::session::StalkerSession; +use crate::iptv::stalker::session::StalkerSession; /// Information the `get_profile` action returns about the underlying portal account. We /// deserialize it loosely (all fields optional) because the field set varies by portal diff --git a/backend/src/utils/network/stalker/recipes.rs b/backend/src/iptv/stalker/recipes.rs similarity index 93% rename from backend/src/utils/network/stalker/recipes.rs rename to backend/src/iptv/stalker/recipes.rs index 11975d83a..4f52a9bf1 100644 --- a/backend/src/utils/network/stalker/recipes.rs +++ b/backend/src/iptv/stalker/recipes.rs @@ -2,7 +2,7 @@ use shared::model::stalker::{ StalkerAuthMode, StalkerBootstrapRecipe, StalkerEndpointPreference, StalkerMagPreset, StalkerPortalFingerprint, }; -use crate::utils::network::stalker::presets::stalker_mag_preset_spec; +use crate::iptv::stalker::presets::stalker_mag_preset_spec; /// A recipe captures "how to perform the handshake against a portal of flavour X using /// auth mode Y and MAG preset Z". The handshake is a sequence of HTTP calls (`handshake`, @@ -153,8 +153,8 @@ pub fn detect_fingerprint( /// The relative order of other candidates is preserved. pub fn apply_endpoint_preference( pref: StalkerEndpointPreference, - mut candidates: Vec, -) -> Vec { + mut candidates: Vec, +) -> Vec { match pref { StalkerEndpointPreference::Auto => candidates, StalkerEndpointPreference::ServerLoad => { @@ -168,7 +168,7 @@ pub fn apply_endpoint_preference( } } -fn rotate_to_front(candidates: &mut Vec, path: &str) { +fn rotate_to_front(candidates: &mut Vec, path: &str) { if let Some(pos) = candidates.iter().position(|c| c.load_url.ends_with(path)) { if pos == 0 { return; @@ -233,17 +233,17 @@ mod tests { assert!(spec.token_in_query); } - fn make_candidates() -> Vec { + fn make_candidates() -> Vec { vec![ - crate::utils::network::stalker::url_factory::StalkerLoadUrl { + crate::iptv::stalker::url_factory::StalkerLoadUrl { load_url: "http://portal.example/server/load.php".to_string(), referer: "http://portal.example/c/".to_string(), }, - crate::utils::network::stalker::url_factory::StalkerLoadUrl { + crate::iptv::stalker::url_factory::StalkerLoadUrl { load_url: "http://portal.example/portal.php".to_string(), referer: "http://portal.example/c/".to_string(), }, - crate::utils::network::stalker::url_factory::StalkerLoadUrl { + crate::iptv::stalker::url_factory::StalkerLoadUrl { load_url: "http://portal.example/c/".to_string(), referer: "http://portal.example/c/".to_string(), }, @@ -280,7 +280,7 @@ mod tests { fn endpoint_preference_missing_path_is_noop() { // No `server/load.php` in the list — rotation must not panic and must keep // the original order. - let cands = vec![crate::utils::network::stalker::url_factory::StalkerLoadUrl { + let cands = vec![crate::iptv::stalker::url_factory::StalkerLoadUrl { load_url: "http://portal.example/portal.php".to_string(), referer: "http://portal.example/c/".to_string(), }]; diff --git a/backend/src/utils/network/stalker/session.rs b/backend/src/iptv/stalker/session.rs similarity index 79% rename from backend/src/utils/network/stalker/session.rs rename to backend/src/iptv/stalker/session.rs index 76db23c77..22ce2b4ab 100644 --- a/backend/src/utils/network/stalker/session.rs +++ b/backend/src/iptv/stalker/session.rs @@ -1,5 +1,7 @@ use std::{fmt, time::{Duration, SystemTime, UNIX_EPOCH}}; +use super::error::safe_stalker_url; + /// A successful Stalker handshake. The token is sent as `Authorization: Bearer ` on /// every subsequent API call; portal cookies live in the client's shared cookie jar; the /// referer is the `Referer` header the portal expects; the `fingerprint_evidence` is a @@ -18,10 +20,12 @@ pub struct StalkerSession { impl fmt::Debug for StalkerSession { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let referer = safe_stalker_url(&self.referer); + let load_url = safe_stalker_url(&self.load_url); f.debug_struct("StalkerSession") .field("token", &"[redacted]") - .field("referer", &self.referer) - .field("load_url", &self.load_url) + .field("referer", &referer) + .field("load_url", &load_url) .field("fingerprint_evidence", &self.fingerprint_evidence) .field("created_at_epoch_secs", &self.created_at_epoch_secs) .finish() @@ -82,6 +86,22 @@ mod tests { assert_eq!(session.fingerprint_evidence.len(), 2); } + #[test] + fn debug_redacts_url_credentials_and_query_secrets() { + let session = StalkerSession::new( + "token-secret".to_string(), + "https://user:pass@portal.example/c/?mac=00:11:22:33:44:55".to_string(), + "https://user:pass@portal.example/server/load.php?token=query-secret".to_string(), + ); + + let debug = format!("{session:?}"); + for secret in ["token-secret", "user", "pass", "00:11:22:33:44:55", "query-secret"] { + assert!(!debug.contains(secret), "debug output leaked {secret}"); + } + assert!(debug.contains("https://portal.example/c/")); + assert!(debug.contains("https://portal.example/server/load.php")); + } + #[test] fn ttl_is_in_fifteen_minute_range() { assert!(STALKER_SESSION_TTL.as_secs() >= 60); diff --git a/backend/src/utils/network/stalker/url_factory.rs b/backend/src/iptv/stalker/url_factory.rs similarity index 97% rename from backend/src/utils/network/stalker/url_factory.rs rename to backend/src/iptv/stalker/url_factory.rs index aad18e50a..689fde1db 100644 --- a/backend/src/utils/network/stalker/url_factory.rs +++ b/backend/src/iptv/stalker/url_factory.rs @@ -1,6 +1,6 @@ use url::Url; -use crate::utils::network::stalker::error::{StalkerError, StalkerResult}; +use crate::iptv::stalker::error::{safe_stalker_url, StalkerError, StalkerResult}; /// The candidate endpoints a Stalker portal might respond on, in priority order. The portal /// will answer on any of these depending on the firmware flavour (legacy MAG250, Ministra, @@ -36,7 +36,7 @@ pub fn load_url_candidates(portal_url: &str) -> StalkerResult { pub user_token: Option<&'a str>, } +fn is_direct_archive_start_query_key(key: &str) -> bool { + key.eq_ignore_ascii_case("utc") || key.eq_ignore_ascii_case("utcstart") +} + +fn is_contextual_archive_start_query_key(key: &str) -> bool { + key.eq_ignore_ascii_case("start") || key.eq_ignore_ascii_case("timestamp") +} + +fn is_archive_start_context_query_key(key: &str) -> bool { + key.eq_ignore_ascii_case("end") + || key.eq_ignore_ascii_case("duration") + || key.eq_ignore_ascii_case("lutc") + || key.eq_ignore_ascii_case("offset") +} + +fn has_archive_start_query(url: &Url) -> bool { + let has_context = url.query_pairs().any(|(key, _)| is_archive_start_context_query_key(&key)); + url.query_pairs().any(|(key, _)| { + is_direct_archive_start_query_key(&key) || (has_context && is_contextual_archive_start_query_key(&key)) + }) +} + +fn preserve_archive_start_query(base: &Url, mut target: Url) -> Url { + let has_context = base.query_pairs().any(|(key, _)| is_archive_start_context_query_key(&key)); + if !has_archive_start_query(base) { + return target; + } + + for (key, value) in base.query_pairs() { + let is_start = is_direct_archive_start_query_key(&key) + || (has_context && is_contextual_archive_start_query_key(&key)); + let already_present = if is_start { + has_archive_start_query(&target) + } else { + target.query_pairs().any(|(target_key, _)| target_key.eq_ignore_ascii_case(&key)) + }; + if (is_start || is_archive_start_context_query_key(&key)) && !already_present { + target.query_pairs_mut().append_pair(&key, &value); + } + } + target +} + +fn preserve_token_query(base: &Url, mut target: Url) -> Url { + let target_has_token = target.query_pairs().any(|(key, _)| key.eq_ignore_ascii_case("token")); + if target_has_token { + return target; + } + + if let Some((key, value)) = base.query_pairs().find(|(key, _)| key.eq_ignore_ascii_case("token")) { + target.query_pairs_mut().append_pair(&key, &value); + } + target +} + +fn has_same_origin(left: &Url, right: &Url) -> bool { + matches!(left.scheme(), "ftp" | "http" | "https" | "ws" | "wss") + && left.scheme() == right.scheme() + && left.host() == right.host() + && left.port_or_known_default() == right.port_or_known_default() +} + /// Rewrites an HLS URI relative to a base playlist URL. /// Absolute URIs are returned unchanged. pub fn rewrite_hls_url<'a>(base: &'a str, reference: &'a str) -> Cow<'a, str> { - // absolute URI -> passthrough if Url::parse(reference).is_ok() { return Cow::Borrowed(reference); } @@ -75,7 +136,23 @@ pub fn rewrite_hls_url<'a>(base: &'a str, reference: &'a str) -> Cow<'a, str> { return Cow::Borrowed(reference); }; - base_url.join(reference).map_or_else(|_| Cow::Borrowed(reference), |u| Cow::Owned(u.to_string())) + base_url.join(reference).map_or_else( + |_| Cow::Borrowed(reference), + |mut target| { + let is_same_origin = has_same_origin(&target, &base_url); + if is_same_origin { + target = preserve_token_query(&base_url, target); + } + let is_same_origin_child_playlist = is_same_origin + && extract_extension_from_url(target.as_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case(HLS_EXT)); + Cow::Owned(if is_same_origin_child_playlist { + preserve_archive_start_query(&base_url, target).into() + } else { + target.into() + }) + }, + ) } fn rewrite_uri_attrib<'a>(line: &'a str, props: &RewriteHlsProps, user: &ProxyUserCredentials) -> Cow<'a, str> { @@ -193,6 +270,113 @@ mod test { assert_eq!(out, "http://example.com/hls/seg001.ts"); } + #[test] + fn rewrite_relative_hls_resources_inherit_token() { + let base = "https://cdn.example/hls/channel/timeshift_abs-1785136500.m3u8?token=secret"; + + for (reference, expected) in [ + ( + "tracks-v1a1/timeshift_abs-1785136500.m3u8", + "https://cdn.example/hls/channel/tracks-v1a1/timeshift_abs-1785136500.m3u8?token=secret", + ), + ( + "tracks-v1a1/dvr-2026/07/27/12/00/00-06000.ts", + "https://cdn.example/hls/channel/tracks-v1a1/dvr-2026/07/27/12/00/00-06000.ts?token=secret", + ), + ("key.bin?version=2", "https://cdn.example/hls/channel/key.bin?version=2&token=secret"), + ("init.mp4", "https://cdn.example/hls/channel/init.mp4?token=secret"), + ] { + assert_eq!(rewrite_hls_url(base, reference), expected); + } + } + + #[test] + fn rewrite_relative_hls_resource_keeps_its_own_token() { + let base = "https://cdn.example/hls/channel/index.m3u8?TOKEN=base"; + + assert_eq!( + rewrite_hls_url(base, "child.m3u8?token=child"), + "https://cdn.example/hls/channel/child.m3u8?token=child" + ); + } + + #[test] + fn rewrite_does_not_leak_token_to_cross_origin_resource() { + let base = "https://cdn.example/hls/channel/index.m3u8?token=secret"; + + assert_eq!(rewrite_hls_url(base, "https://media.example/child.m3u8"), "https://media.example/child.m3u8"); + assert_eq!(rewrite_hls_url(base, "//media.example/child.m3u8"), "https://media.example/child.m3u8"); + } + + #[test] + fn rewrite_relative_variant_preserves_archive_start_query() { + let base = "https://cdn.example/hls/channel/index.m3u8?offset=-10752&utcstart=1785072000&useseq=t"; + let uri = "variant/playlist.m3u8?offset=-10752&useseq=t"; + + let out = rewrite_hls_url(base, uri); + assert_eq!( + out, + "https://cdn.example/hls/channel/variant/playlist.m3u8?offset=-10752&useseq=t&utcstart=1785072000" + ); + } + + #[test] + fn rewrite_relative_variant_preserves_all_archive_context_queries() { + let base = "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600&end=1785075600&duration=3600"; + let uri = "variant/playlist.m3u8?offset=-1800"; + + let out = rewrite_hls_url(base, uri); + assert_eq!( + out, + "https://cdn.example/hls/channel/variant/playlist.m3u8?offset=-1800&utcstart=1785072000&end=1785075600&duration=3600" + ); + } + + #[test] + fn rewrite_keeps_child_archive_start_query() { + let base = "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000"; + let uri = "variant/playlist.m3u8?utc=1785071000"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "https://cdn.example/hls/channel/variant/playlist.m3u8?utc=1785071000"); + } + + #[test] + fn rewrite_does_not_propagate_plain_start_query() { + let base = "https://cdn.example/hls/channel/index.m3u8?start=1785072000"; + let uri = "variant/playlist.m3u8"; + + let out = rewrite_hls_url(base, uri); + assert_eq!(out, "https://cdn.example/hls/channel/variant/playlist.m3u8"); + } + + #[test] + fn rewrite_archive_playlist_does_not_modify_signed_media_urls() { + let base = "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600"; + + assert_eq!( + rewrite_hls_url(base, "segment.ts?sig=abc"), + "https://cdn.example/hls/channel/segment.ts?sig=abc" + ); + assert_eq!( + rewrite_hls_url(base, "key.bin?sig=def"), + "https://cdn.example/hls/channel/key.bin?sig=def" + ); + assert_eq!( + rewrite_hls_url(base, "init.mp4?sig=ghi"), + "https://cdn.example/hls/channel/init.mp4?sig=ghi" + ); + } + + #[test] + fn rewrite_absolute_same_origin_url_remains_exact_passthrough() { + let base = "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600"; + let reference = "https://cdn.example/hls/channel/segment.ts?sig=a%2Fb&token=x+y"; + + assert!(matches!(rewrite_hls_url(base, reference), std::borrow::Cow::Borrowed(_))); + assert_eq!(rewrite_hls_url(base, reference), reference); + } + #[test] fn rewrite_https_absolute_passthrough() { let base = "http://example.com/hls/playlist.m3u8"; @@ -220,6 +404,13 @@ mod test { assert_eq!(out, "file:///mnt/media/hls/seg001.ts"); } + #[test] + fn rewrite_file_child_playlist_does_not_inherit_archive_query() { + let base = "file:///mnt/media/hls/playlist.m3u8?utc=1785072000"; + + assert_eq!(rewrite_hls_url(base, "child.m3u8"), "file:///mnt/media/hls/child.m3u8"); + } + #[test] fn rewrite_file_absolute_passthrough() { let base = "file:///mnt/media/hls/playlist.m3u8"; diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 973601280..33684f9c3 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -462,9 +462,22 @@ fn parse_extm3u_catchup_correction(attributes: &str) -> Option> { None } +fn parse_extvlcopt_user_agent(line: &str) -> Option<&str> { + let (prefix, option) = line.trim().split_once(':')?; + if !prefix.eq_ignore_ascii_case("#EXTVLCOPT") { + return None; + } + let (name, value) = option.split_once('=')?; + if !name.trim().eq_ignore_ascii_case("http-user-agent") { + return None; + } + Some(value.trim()) +} + pub async fn consume_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader, mut visit: F) { let mut header: Option = None; let mut group: Option> = None; + let mut upstream_user_agent: Option> = None; let mut default_catchup_correction: Option> = None; let input_name = &input.name; @@ -481,6 +494,13 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp if let Some(b'#') = bytes.first().copied() { if bytes.starts_with(b"#EXTINF") { header = Some(line); + upstream_user_agent = None; + continue; + } + if let Some(value) = parse_extvlcopt_user_agent(&line) { + if header.is_some() { + upstream_user_agent = (!value.is_empty()).then(|| value.intern()); + } continue; } if let Some(rest) = line.strip_prefix("#EXTM3U") { @@ -505,6 +525,7 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp ), }; let header = &mut item.header; + header.upstream_user_agent = upstream_user_agent.take(); header.source_ordinal = ord_counter; ord_counter += 1; if header.group.is_empty() { @@ -832,4 +853,28 @@ mod test { assert_eq!(items[0].header.group.as_ref(), "Sports"); } + #[tokio::test] + async fn consume_m3u_scopes_extvlcopt_user_agent_to_one_item() { + let content = concat!( + "#EXTM3U\n", + "#EXTINF:-1,Channel 1\n", + "#extvlcopt:HTTP-USER-AGENT= Source UA/1.0 \n", + "http://provider.example/live/user/pass/1.ts\n", + "#EXTINF:-1,Channel 2\n", + "#EXTVLCOPT:http-user-agent=\n", + "http://provider.example/live/user/pass/2.ts\n", + "#EXTVLCOPT:http-user-agent=must-not-leak\n", + "#EXTINF:-1,Channel 3\n", + "http://provider.example/live/user/pass/3.ts\n", + ); + let mut items = Vec::new(); + + super::consume_m3u(&Config::default(), &test_input(), make_reader(content), |item| items.push(item)).await; + + assert_eq!(items.len(), 3); + assert_eq!(items[0].header.upstream_user_agent.as_deref(), Some("Source UA/1.0")); + assert_eq!(items[1].header.upstream_user_agent, None); + assert_eq!(items[2].header.upstream_user_agent, None); + } + } diff --git a/backend/src/processing/parser/stalker.rs b/backend/src/processing/parser/stalker.rs index 91375ef2b..3c5c4e5b4 100644 --- a/backend/src/processing/parser/stalker.rs +++ b/backend/src/processing/parser/stalker.rs @@ -21,7 +21,7 @@ use shared::utils::Internable; use std::collections::HashSet; use std::sync::Arc; -use crate::utils::network::stalker::catalog::{ +use crate::iptv::stalker::catalog::{ StalkerCategory, StalkerRawItem, StalkerRawItemInfo, StalkerRawSeriesDetails, StalkerRawSeriesEpisode, StalkerRawSeriesItem, StalkerRawSeriesSeason, }; diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index c5f3f3f01..531bd831a 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -1,7 +1,7 @@ use crate::model::{ConfigInput, ConfigInputFlags}; use crate::model::XtreamCategory; use crate::utils::request::DynReader; -use crate::utils::xtream::get_xtream_stream_url_base; +use crate::iptv::xtream::get_xtream_stream_url_base; use indexmap::IndexMap; use serde::Deserializer; use shared::error::TuliproxError; diff --git a/backend/src/processing/processor/playlist.rs b/backend/src/processing/processor/playlist.rs index 0554d5581..41f4a975e 100644 --- a/backend/src/processing/processor/playlist.rs +++ b/backend/src/processing/processor/playlist.rs @@ -6,6 +6,7 @@ use crate::{ }, sync_panel_api_exp_dates, }, + iptv::{m3u, xtream}, messaging::send_message, media_server::{ media_server_catalog_snapshot_to_playlist, refresh_media_server_catalog_complete_before_publish, @@ -32,7 +33,7 @@ use crate::{ PlaylistSource, }, utils::{ - debug_if_enabled, epg, log_memory_snapshot, m3u, trace_if_enabled, xtream, + debug_if_enabled, epg, log_memory_snapshot, trace_if_enabled, StepMeasure, StepMeasureCallback, }, }; @@ -2077,18 +2078,29 @@ mod tests { use shared::utils::Internable; use crate::model::Config; - fn serialize_without_trailing_input_stream_id(value: &T) -> Vec { + fn serialize_without_trailing_fields(value: &T, trailing_fields: &[u8]) -> Vec { let mut encoded = rmp_serde::to_vec(value).expect("playlist item should serialize"); - assert_eq!(encoded.pop(), Some(0xa0), "input_stream_id should be the trailing empty string"); + for expected in trailing_fields { + assert_eq!(encoded.pop(), Some(*expected), "unexpected trailing MessagePack field"); + } + let removed = trailing_fields.len(); match encoded[0] { - marker @ 0x91..=0x9f => encoded[0] = marker - 1, + marker @ 0x92..=0x9f => { + let len = usize::from(marker - 0x90); + assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); + encoded[0] = 0x90 + u8::try_from(len - removed).unwrap_or_default(); + } 0xdc => { let len = u16::from_be_bytes([encoded[1], encoded[2]]); - encoded[1..3].copy_from_slice(&(len - 1).to_be_bytes()); + let removed = u16::try_from(removed).unwrap_or(u16::MAX); + assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); + encoded[1..3].copy_from_slice(&(len - removed).to_be_bytes()); } 0xdd => { let len = u32::from_be_bytes([encoded[1], encoded[2], encoded[3], encoded[4]]); - encoded[1..5].copy_from_slice(&(len - 1).to_be_bytes()); + let removed = u32::try_from(removed).unwrap_or(u32::MAX); + assert!(len >= removed, "trailing field count exceeds MessagePack sequence length"); + encoded[1..5].copy_from_slice(&(len - removed).to_be_bytes()); } marker => panic!("unexpected MessagePack sequence marker {marker:#x}"), } @@ -2231,11 +2243,12 @@ mod tests { }, }; - let header_bytes = serialize_without_trailing_input_stream_id(&source.header); + let header_bytes = serialize_without_trailing_fields(&source.header, &[0xc0, 0xa0]); let decoded_header: PlaylistItemHeader = rmp_serde::from_slice(&header_bytes).expect("legacy header should deserialize"); assert!(decoded_header.input_stream_id.is_empty()); assert_eq!(decoded_header.get_input_stream_id(), None); + assert_eq!(decoded_header.upstream_user_agent, None); let mut decoded_header = decoded_header; decoded_header.freeze_input_stream_id(); assert_eq!(decoded_header.get_input_stream_id().as_deref(), Some("origin-alpha")); @@ -2243,19 +2256,73 @@ mod tests { source.header.freeze_input_stream_id(); let mut m3u_item = M3uPlaylistItem::from(&source); m3u_item.input_stream_id = "".intern(); - let m3u_bytes = serialize_without_trailing_input_stream_id(&m3u_item); + let m3u_bytes = serialize_without_trailing_fields(&m3u_item, &[0xc0, 0xa0]); let decoded_m3u: M3uPlaylistItem = rmp_serde::from_slice(&m3u_bytes).expect("legacy M3U item should deserialize"); assert!(decoded_m3u.input_stream_id.is_empty()); assert_eq!(decoded_m3u.get_input_stream_id().as_deref(), Some("origin-alpha")); + assert_eq!(decoded_m3u.upstream_user_agent, None); let mut xtream_item = XtreamPlaylistItem::from(&source); xtream_item.input_stream_id = "".intern(); - let xtream_bytes = serialize_without_trailing_input_stream_id(&xtream_item); + let xtream_bytes = serialize_without_trailing_fields(&xtream_item, &[0xc0, 0xa0]); let decoded_xtream: XtreamPlaylistItem = rmp_serde::from_slice(&xtream_bytes).expect("legacy Xtream item should deserialize"); assert!(decoded_xtream.input_stream_id.is_empty()); assert_eq!(decoded_xtream.get_input_stream_id().as_deref(), Some("80510")); + assert_eq!(decoded_xtream.upstream_user_agent, None); + } + + #[test] + fn previous_messagepack_playlist_items_default_missing_upstream_user_agent() { + let source = PlaylistItem { + header: PlaylistItemHeader { + id: "80510".intern(), + input_stream_id: "origin-alpha".intern(), + ..Default::default() + }, + }; + + let header: PlaylistItemHeader = rmp_serde::from_slice(&serialize_without_trailing_fields(&source.header, &[0xc0])) + .expect("previous header should deserialize"); + let m3u: M3uPlaylistItem = rmp_serde::from_slice(&serialize_without_trailing_fields( + &M3uPlaylistItem::from(&source), + &[0xc0], + )) + .expect("previous M3U item should deserialize"); + let xtream: XtreamPlaylistItem = rmp_serde::from_slice(&serialize_without_trailing_fields( + &XtreamPlaylistItem::from(&source), + &[0xc0], + )) + .expect("previous Xtream item should deserialize"); + + assert_eq!(header.input_stream_id.as_ref(), "origin-alpha"); + assert_eq!(m3u.input_stream_id.as_ref(), "origin-alpha"); + assert_eq!(xtream.input_stream_id.as_ref(), "origin-alpha"); + assert_eq!(header.upstream_user_agent, None); + assert_eq!(m3u.upstream_user_agent, None); + assert_eq!(xtream.upstream_user_agent, None); + } + + #[test] + fn messagepack_playlist_items_preserve_upstream_user_agent() -> Result<(), Box> { + let source = PlaylistItem { + header: PlaylistItemHeader { + upstream_user_agent: Some("Provider-UA".intern()), + ..Default::default() + }, + }; + + let header: PlaylistItemHeader = rmp_serde::from_slice(&rmp_serde::to_vec(&source.header)?)?; + let m3u: M3uPlaylistItem = + rmp_serde::from_slice(&rmp_serde::to_vec(&M3uPlaylistItem::from(&source))?)?; + let xtream: XtreamPlaylistItem = + rmp_serde::from_slice(&rmp_serde::to_vec(&XtreamPlaylistItem::from(&source))?)?; + + assert_eq!(header.upstream_user_agent.as_deref(), Some("Provider-UA")); + assert_eq!(m3u.upstream_user_agent.as_deref(), Some("Provider-UA")); + assert_eq!(xtream.upstream_user_agent.as_deref(), Some("Provider-UA")); + Ok(()) } #[test] diff --git a/backend/src/processing/processor/stalker.rs b/backend/src/processing/processor/stalker.rs index f9e8bed81..e7579f644 100644 --- a/backend/src/processing/processor/stalker.rs +++ b/backend/src/processing/processor/stalker.rs @@ -6,7 +6,7 @@ //! //! Reverse-proxy re-resolve (when a 4xx upstream error is observed) is implemented in the //! HLS/Xtream endpoints and reaches back into the API client via the helper -//! [`crate::utils::network::stalker::client::StalkerApiClient::create_link`]. +//! [`crate::iptv::stalker::client::StalkerApiClient::create_link`]. #![allow(clippy::too_many_lines, clippy::needless_pass_by_value)] @@ -33,8 +33,8 @@ use crate::repository::stalker_generation_repository::{load_active_manifest, loa use super::stalker_refresh::{ advance_stalker_refresh, StalkerClusterSelection, StalkerRefreshMode, StalkerRefreshOutcome, }; -use crate::utils::network::stalker::client::StalkerApiClient; -use crate::utils::network::stalker::error::StalkerError; +use crate::iptv::stalker::client::StalkerApiClient; +use crate::iptv::stalker::error::StalkerError; /// Cluster selector used by the Stalker processor. Mirrors the Xtream cluster split diff --git a/backend/src/processing/processor/stalker_refresh.rs b/backend/src/processing/processor/stalker_refresh.rs index fc5221a94..9f2a565e1 100644 --- a/backend/src/processing/processor/stalker_refresh.rs +++ b/backend/src/processing/processor/stalker_refresh.rs @@ -18,10 +18,10 @@ use crate::repository::stalker_repository::{ remove_stalker_file, snapshot_stalker_epg_at, snapshot_stalker_items_at, upsert_stalker_epg_at, upsert_stalker_items_at, }; -use crate::utils::network::stalker::catalog::{StalkerCategory, StalkerRawItem}; -use crate::utils::network::stalker::client::StalkerApiClient; -use crate::utils::network::stalker::error::StalkerError; -use crate::utils::network::stalker::profile::StalkerHandshake; +use crate::iptv::stalker::catalog::{StalkerCategory, StalkerRawItem}; +use crate::iptv::stalker::client::StalkerApiClient; +use crate::iptv::stalker::error::StalkerError; +use crate::iptv::stalker::profile::StalkerHandshake; use super::stalker::StalkerCluster; const MAX_RETRIES: u8 = 3; diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index ef8fe0c8d..58aa86b53 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -1,6 +1,7 @@ use crate::api::model::UpdateTask; use crate::api::model::{ActiveProviderManager, ProviderHandle, ProviderIdType, ResolveReason, ResolveReasonSet}; use crate::library::{MetadataResolver, MetadataStorage}; +use crate::iptv::xtream; use crate::media_enrichment::policy::MissingFactEnrichmentPolicy; use crate::media_enrichment::xtream::{ apply_fact_patch_to_series, series_fact_patch_from_metadata, series_fact_patch_from_title_candidates, @@ -21,7 +22,7 @@ use crate::repository::persists_input_series_info; use crate::repository::{get_input_storage_path, persist_input_series_info_batch, MemoryPlaylistSource}; use crate::repository::{xtream_get_file_path, BPlusTreeQuery}; use crate::utils::ffmpeg::{is_supported_probe_url, FfmpegExecutor, ProbeFailureKind, ProbeUrlOutcome}; -use crate::utils::{debug_if_enabled, xtream}; +use crate::utils::debug_if_enabled; use log::{debug, error, info, log_enabled, trace, warn, Level}; use parking_lot::Mutex; use serde_json::Value; diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 7f0306b76..7561b2129 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -1,6 +1,7 @@ use crate::api::model::{ActiveProviderManager, ProviderHandle}; use crate::api::model::{ProviderIdType, ResolveReason, ResolveReasonSet, UpdateTask}; use crate::library::{MediaMetadata, MetadataResolver, MetadataStorage}; +use crate::iptv::xtream; use crate::media_enrichment::policy::MissingFactEnrichmentPolicy; use crate::media_enrichment::xtream::{ apply_fact_patch_to_video, distinct_non_empty_title_candidates, video_fact_patch_from_metadata, @@ -22,7 +23,7 @@ use crate::repository::persist_input_vod_info; use crate::repository::persist_input_vod_info_batch; use crate::repository::{xtream_get_file_path, BPlusTreeQuery}; use crate::utils::ffmpeg::{is_supported_probe_url, FfmpegExecutor, ProbeFailureKind, ProbeUrlOutcome}; -use crate::utils::{debug_if_enabled, trace_if_enabled, xtream}; +use crate::utils::{debug_if_enabled, trace_if_enabled}; use log::{debug, error, info, log_enabled, trace, warn, Level}; use parking_lot::Mutex; use serde_json::Value; diff --git a/backend/src/repository/library_repository.rs b/backend/src/repository/library_repository.rs index acd7ce817..433a44c34 100644 --- a/backend/src/repository/library_repository.rs +++ b/backend/src/repository/library_repository.rs @@ -205,6 +205,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "1".intern(), + upstream_user_agent: None, } } @@ -247,6 +248,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "2".intern(), + upstream_user_agent: None, } } diff --git a/backend/src/repository/m3u_playlist_iterator.rs b/backend/src/repository/m3u_playlist_iterator.rs index 9f8143367..6a5d7eb5f 100644 --- a/backend/src/repository/m3u_playlist_iterator.rs +++ b/backend/src/repository/m3u_playlist_iterator.rs @@ -1,11 +1,11 @@ use crate::model::ConfigTarget; use crate::model::{AppConfig, ProxyUserCredentials}; +use crate::iptv::m3u::{build_m3u_catchup_rewrite, flussonic_proxy_live_file}; use crate::repository::m3u_get_file_path_for_db; use crate::repository::storage_const; use crate::repository::user_get_bouquet_filter; use crate::repository::{ensure_target_storage_path, get_file_path_for_db_index}; use crate::repository::{open_playlist_reader, LockedReceiverStream}; -use crate::utils::build_m3u_catchup_rewrite; use futures::Stream; use log::error; use shared::create_bitset; @@ -163,13 +163,36 @@ fn apply_rewrite( }; if should_rewrite_urls { - let stream_url = build_rewritten_url( - ctx, - effective_source_url.as_ref(), - &m3u_pli, - flags.contains(M3uPlaylistIteratorFlags::IncludeTypeInUrl), - storage_const::M3U_STREAM_PATH, - true, + let flussonic_live_file = m3u_pli.additional_properties.as_ref().and_then(|properties| match properties { + StreamProperties::Live(live) => live + .catchup + .as_ref() + .and_then(shared::model::CatchupProperties::native_flussonic_player_mode) + .map(flussonic_proxy_live_file), + _ => None, + }); + let stream_url = flussonic_live_file.map_or_else( + || { + build_rewritten_url( + ctx, + effective_source_url.as_ref(), + &m3u_pli, + flags.contains(M3uPlaylistIteratorFlags::IncludeTypeInUrl), + storage_const::M3U_STREAM_PATH, + true, + ) + }, + |live_file| { + let base = build_rewritten_url( + ctx, + effective_source_url.as_ref(), + &m3u_pli, + flags.contains(M3uPlaylistIteratorFlags::IncludeTypeInUrl), + storage_const::M3U_STREAM_PATH, + false, + ); + shared::concat_string!(&base, "/", live_file) + }, ); let resource_url = if flags.contains(M3uPlaylistIteratorFlags::RewriteResource) { let source_url = if m3u_pli.logo.is_empty() { m3u_pli.logo_small.as_ref() } else { m3u_pli.logo.as_ref() }; @@ -198,6 +221,9 @@ fn apply_rewrite( m3u_pli.t_catchup_mode = Some(rewrite.mode); m3u_pli.t_catchup_source = Some(rewrite.source); } + if should_rewrite_urls { + m3u_pli.upstream_user_agent = None; + } m3u_pli } @@ -350,10 +376,23 @@ impl Stream for M3uPlaylistIterator { } } +fn build_proxy_xmltv_url_tvg(base_url: &str, username: &str, password: &str) -> Option { + let base = base_url.trim_end_matches('/'); + if base.is_empty() || username.is_empty() || password.is_empty() { + return None; + } + let query = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("username", username) + .append_pair("password", password) + .finish(); + Some(format!("{base}/xmltv.php?{query}")) +} + pub struct M3uPlaylistM3uTextIterator { inner: M3uPlaylistIterator, started: bool, target_options: Option, + url_tvg: Option, } impl M3uPlaylistM3uTextIterator { @@ -362,10 +401,12 @@ impl M3uPlaylistM3uTextIterator { target: &ConfigTarget, user: &ProxyUserCredentials, ) -> Result { + let base_url = cfg.get_user_server_info(user).map(|server| server.get_base_url()).unwrap_or_default(); Ok(Self { inner: M3uPlaylistIterator::new(cfg, target, user).await?, started: false, target_options: target.options.clone(), + url_tvg: build_proxy_xmltv_url_tvg(&base_url, &user.username, &user.password), }) } } @@ -376,7 +417,11 @@ impl Stream for M3uPlaylistM3uTextIterator { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { if !self.started { self.started = true; - return Poll::Ready(Some(Ok("#EXTM3U".to_string()))); + let header = self.url_tvg.as_ref().map_or_else( + || "#EXTM3U".to_string(), + |url| format!("#EXTM3U url-tvg=\"{url}\" x-tvg-url=\"{url}\""), + ); + return Poll::Ready(Some(Ok(header))); } match Pin::new(&mut self.inner).poll_next(cx) { @@ -394,14 +439,17 @@ impl Stream for M3uPlaylistM3uTextIterator { #[cfg(test)] mod tests { use super::{ - apply_rewrite, M3uPlaylistIterator, M3uPlaylistIteratorFlags, M3uPlaylistIteratorFlagsSet, - M3uPlaylistM3uTextIterator, UrlRewriteContext, + apply_rewrite, build_proxy_xmltv_url_tvg, M3uPlaylistIterator, M3uPlaylistIteratorFlags, + M3uPlaylistIteratorFlagsSet, M3uPlaylistM3uTextIterator, UrlRewriteContext, }; use crate::model::{ConfigInput, ConfigProvider, ProviderDnsCache}; use crate::repository::LockedReceiverStream; use futures::StreamExt; use shared::{ - model::{M3uPlaylistItem, PlaylistItemType, ProviderUrlSelectionPolicy, ProxyType}, + model::{ + CatchupProperties, LiveStreamProperties, M3uPlaylistItem, PlaylistItemType, ProviderUrlSelectionPolicy, + ProxyType, StreamProperties, + }, utils::Internable, }; use std::{ @@ -450,6 +498,7 @@ mod tests { source_ordinal: 0, additional_properties: None, input_stream_id: "813294".intern(), + upstream_user_agent: None, } } @@ -498,6 +547,55 @@ mod tests { assert_eq!(rewritten.t_stream_url.as_ref(), "http://example.com/live/user/pass/813294.ts"); } + #[test] + fn upstream_user_agent_is_only_exposed_for_direct_provider_urls() { + let input_by_name = HashMap::new(); + let ctx = UrlRewriteContext { + base_url: "https://example.com", + username: "user", + password: "pass", + }; + let mut direct_item = m3u_item("http://example.com/live/user/pass/813294.ts"); + direct_item.upstream_user_agent = Some("Provider-UA".intern()); + + let direct = apply_rewrite( + direct_item.clone(), + &ctx, + 1, + &[7u8; 16], + &input_by_name, + None, + M3uPlaylistIteratorFlagsSet::new(), + ProxyType::Redirect, + ); + let reverse = apply_rewrite( + direct_item, + &ctx, + 1, + &[7u8; 16], + &input_by_name, + None, + M3uPlaylistIteratorFlagsSet::new(), + ProxyType::Reverse(None), + ); + let mut masked_flags = M3uPlaylistIteratorFlagsSet::new(); + masked_flags.set(M3uPlaylistIteratorFlags::MaskRedirectUrl); + let masked_redirect = apply_rewrite( + direct.clone(), + &ctx, + 1, + &[7u8; 16], + &input_by_name, + None, + masked_flags, + ProxyType::Redirect, + ); + + assert!(direct.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=Provider-UA")); + assert!(!reverse.to_m3u(None, true).contains("#EXTVLCOPT:http-user-agent")); + assert!(!masked_redirect.to_m3u(None, true).contains("#EXTVLCOPT:http-user-agent")); + } + #[test] fn redirect_without_mask_preserves_provider_scheme_url_when_input_is_missing() { let input_by_name = HashMap::new(); @@ -545,19 +643,92 @@ mod tests { assert_eq!(rewritten.t_stream_url.as_ref(), "https://example.com/m3u-stream/user/pass/813294.ts"); } - // Regression lock: the streaming M3U response used by `m3u_api` must emit a - // bare `#EXTM3U` header line, never a proxy-rewritten `url-tvg` from the - // source playlist. The parser does not capture the source's `url-tvg` - // attribute (`M3uPlaylistItem` has no field for it), and the writer must - // therefore emit the bare marker regardless of what the upstream feed - // declared. The inner channel is closed immediately so the iterator yields - // no item lines, which lets the assertion focus on the header line only. + #[test] + fn explicit_flussonic_modes_use_native_hls_and_ts_live_paths() { + let ctx = UrlRewriteContext { + base_url: "https://example.com", + username: "user", + password: "pass", + }; + for (mode, source_url, expected_suffix) in [ + ("flussonic", "http://provider/ch/index.m3u8", "/813294/index.m3u8"), + ("flussonic-ts", "http://provider/ch/channel.ts", "/813294/mpegts"), + ] { + let mut item = m3u_item(source_url); + item.additional_properties = Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + catchup_type: Some(mode.intern()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))); + let rewritten = apply_rewrite( + item, + &ctx, + 1, + &[7u8; 16], + &HashMap::new(), + None, + M3uPlaylistIteratorFlagsSet::new(), + ProxyType::Reverse(None), + ); + assert!(rewritten.t_stream_url.ends_with(expected_suffix)); + assert!(rewritten.t_catchup_source.is_none()); + } + } + + #[test] + fn append_mode_keeps_existing_flat_live_path() { + let mut item = m3u_item("http://provider/ch/index.m3u8"); + item.additional_properties = Some(StreamProperties::Live(Box::new(LiveStreamProperties { + catchup: Some(CatchupProperties { + mode: Some("append".intern()), + source: Some("?utcstart=${timestamp}&offset=-${offset}".intern()), + ..CatchupProperties::default() + }), + ..LiveStreamProperties::default() + }))); + let rewritten = apply_rewrite( + item, + &UrlRewriteContext { + base_url: "https://example.com", + username: "user", + password: "pass", + }, + 1, + &[7u8; 16], + &HashMap::new(), + None, + M3uPlaylistIteratorFlagsSet::new(), + ProxyType::Reverse(None), + ); + + assert_eq!(rewritten.t_stream_url.as_ref(), "https://example.com/m3u-stream/user/pass/813294.m3u8"); + assert!(rewritten.t_catchup_source.is_some()); + } + + #[test] + fn proxy_xmltv_url_uses_user_server_path_and_encoded_credentials() { + assert_eq!( + build_proxy_xmltv_url_tvg("https://proxy.example/iptv/", "u ser", "p&ss").as_deref(), + Some("https://proxy.example/iptv/xmltv.php?username=u+ser&password=p%26ss") + ); + assert_eq!(build_proxy_xmltv_url_tvg("", "user", "pass"), None); + } + + // Regression lock: never forward the provider's source `url-tvg`. Without + // configured Tuliprox server information, the header stays bare. #[tokio::test] async fn m3u_text_iterator_emits_bare_extm3u_header_without_proxying_source_url_tvg() { let (tx, rx) = mpsc::channel::>(1); drop(tx); let inner_iter = M3uPlaylistIterator { inner: LockedReceiverStream::new_empty(rx) }; - let mut text_iter = M3uPlaylistM3uTextIterator { inner: inner_iter, started: false, target_options: None }; + let mut text_iter = M3uPlaylistM3uTextIterator { + inner: inner_iter, + started: false, + target_options: None, + url_tvg: None, + }; let first = text_iter.next().await; assert_eq!(first.as_ref().and_then(|line| line.as_deref().ok()), Some("#EXTM3U"), "EXTM3U header must be bare, never contain url-tvg"); @@ -566,6 +737,27 @@ mod tests { assert_eq!(second, None, "inner channel is closed, so no item lines should follow"); } + #[tokio::test] + async fn m3u_text_iterator_emits_proxy_xmltv_header_when_configured() { + let (tx, rx) = mpsc::channel::>(1); + drop(tx); + let inner_iter = M3uPlaylistIterator { inner: LockedReceiverStream::new_empty(rx) }; + let mut text_iter = M3uPlaylistM3uTextIterator { + inner: inner_iter, + started: false, + target_options: None, + url_tvg: Some("https://proxy.example/xmltv.php?username=user&password=pass".to_string()), + }; + + let first = text_iter.next().await; + assert_eq!( + first.as_ref().and_then(|line| line.as_deref().ok()), + Some( + "#EXTM3U url-tvg=\"https://proxy.example/xmltv.php?username=user&password=pass\" x-tvg-url=\"https://proxy.example/xmltv.php?username=user&password=pass\"" + ) + ); + } + #[tokio::test] async fn iterator_forwards_one_storage_error_then_ends() { let (tx, rx) = mpsc::channel(2); diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 3ab8bbae3..b4d644c2b 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -1269,6 +1269,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "9001".intern(), + upstream_user_agent: None, }; let provider_parent_code = xtream_series_info.get_uuid().intern(); let xtream_provider_episode = XtreamPlaylistItem { @@ -1291,6 +1292,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "201".intern(), + upstream_user_agent: None, }; let provider_episode = PlaylistItem::from(&xtream_provider_episode); let mut series_info = PlaylistItem::from(&xtream_series_info); diff --git a/backend/src/repository/stalker_repository.rs b/backend/src/repository/stalker_repository.rs index 602ea2a82..6be036b59 100644 --- a/backend/src/repository/stalker_repository.rs +++ b/backend/src/repository/stalker_repository.rs @@ -23,7 +23,7 @@ use tokio_stream::wrappers::ReceiverStream; use shared::error::TuliproxError; use shared::model::stalker_item::{StalkerPlaylistItem, StalkerSeasonItem}; -use crate::utils::network::stalker::epg::StalkerProgramRecord; +use crate::iptv::stalker::epg::StalkerProgramRecord; use crate::api::model::AppState; use crate::model::{AppConfig, ConfigInput, ConfigTarget}; diff --git a/backend/src/repository/stream_history/mod.rs b/backend/src/repository/stream_history/mod.rs index 747069820..0fc1fbe95 100644 --- a/backend/src/repository/stream_history/mod.rs +++ b/backend/src/repository/stream_history/mod.rs @@ -139,6 +139,7 @@ pub(crate) mod tests { }), epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }, user_agent: String::from("VLC/3.0"), country_code: Some(String::from("DE")), diff --git a/backend/src/repository/xtream_playlist_iterator.rs b/backend/src/repository/xtream_playlist_iterator.rs index 504aad021..7e68b6e64 100644 --- a/backend/src/repository/xtream_playlist_iterator.rs +++ b/backend/src/repository/xtream_playlist_iterator.rs @@ -276,6 +276,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "1".intern(), + upstream_user_agent: None, }; assert!(tx.send(Ok((item, true))).await.is_ok()); assert!(tx.send(Err(shared::error::TuliproxError::RepositoryXtream("corrupt page".into()))).await.is_ok()); diff --git a/backend/src/repository/xtream_repository.rs b/backend/src/repository/xtream_repository.rs index 1189883ab..b4792013c 100644 --- a/backend/src/repository/xtream_repository.rs +++ b/backend/src/repository/xtream_repository.rs @@ -1734,6 +1734,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: provider_id.to_string().intern(), + upstream_user_agent: None, } } diff --git a/backend/src/utils/db_viewer.rs b/backend/src/utils/db_viewer.rs index 193c47290..41d2d20fb 100644 --- a/backend/src/utils/db_viewer.rs +++ b/backend/src/utils/db_viewer.rs @@ -344,6 +344,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "52568".intern(), + upstream_user_agent: None, }; let value = to_human_readable_json_value(&item).expect("dump value should serialize"); diff --git a/backend/src/utils/m3u_catchup.rs b/backend/src/utils/m3u_catchup.rs deleted file mode 100644 index c766443c7..000000000 --- a/backend/src/utils/m3u_catchup.rs +++ /dev/null @@ -1,532 +0,0 @@ -use crate::utils::{encode_m3u_catchup_token, M3uCatchupToken}; -use shared::{ - error::TuliproxError, - model::CatchupProperties, - utils::{short_hash, Internable}, -}; -use std::{borrow::Cow, sync::Arc}; -use url::Url; - -pub const M3U_CATCHUP_ROUTE_PREFIX: &str = "m3u-catchup"; -pub const M3U_CATCHUP_MARKER: &str = "tuliprox-catchup"; -const COLLECTOR_PREFIX: &str = "v"; -const SIPTV_APPEND_TEMPLATE: &str = "?utc={utc}&lutc={lutc}"; -const XC_START_TEMPLATE: &str = "{Y}-{m}-{d}:{H}-{M}-{S}"; -const FLUSSONIC_TEMPLATE: &str = "timeshift_abs-{utc}"; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct M3uCatchupRewrite { - pub mode: Arc, - pub source: Arc, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ResolvedM3uCatchup { - pub url: String, - pub discriminator: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum TemplateSegment { - Literal(String), - Placeholder(String), -} - -fn mode_alias(mode: &str) -> &str { - if mode.eq_ignore_ascii_case("append") { - "append" - } else if mode.eq_ignore_ascii_case("default") { - "default" - } else if mode.eq_ignore_ascii_case("shift") { - "shift" - } else if mode.eq_ignore_ascii_case("xc") { - "xc" - } else if mode.eq_ignore_ascii_case("fs") { - "fs" - } else if mode.eq_ignore_ascii_case("vod") { - "vod" - } else { - mode - } -} - -fn parse_template(template: &str) -> Vec { - let mut segments = Vec::new(); - let mut current = String::new(); - let bytes = template.as_bytes(); - let mut idx = 0usize; - - while idx < bytes.len() { - let start = if bytes[idx] == b'{' { - Some((idx, 1usize)) - } else if idx + 1 < bytes.len() && bytes[idx] == b'$' && bytes[idx + 1] == b'{' { - Some((idx, 2usize)) - } else { - None - }; - - let Some((placeholder_start, open_len)) = start else { - current.push(bytes[idx] as char); - idx += 1; - continue; - }; - - if !current.is_empty() { - segments.push(TemplateSegment::Literal(std::mem::take(&mut current))); - } - - let mut end = placeholder_start + open_len; - while end < bytes.len() && bytes[end] != b'}' { - end += 1; - } - if end >= bytes.len() { - current.push_str(&template[placeholder_start..]); - break; - } - - segments.push(TemplateSegment::Placeholder(template[placeholder_start..=end].to_string())); - idx = end + 1; - } - - if !current.is_empty() { - segments.push(TemplateSegment::Literal(current)); - } - - segments -} - -fn collect_placeholders(segments: &[TemplateSegment]) -> Vec<&str> { - segments - .iter() - .filter_map(|segment| match segment { - TemplateSegment::Placeholder(value) => Some(value.as_str()), - TemplateSegment::Literal(_) => None, - }) - .collect() -} - -fn build_local_source( - base_url: &str, - path: &str, - token: &str, - placeholders: &[&str], - append_mode: bool, -) -> String { - let mut source = if append_mode { - format!("?{M3U_CATCHUP_MARKER}={token}") - } else { - format!("{base_url}/{M3U_CATCHUP_ROUTE_PREFIX}/{token}") - }; - if !append_mode && !path.is_empty() { - source.push('/'); - source.push_str(path.trim_start_matches('/')); - } - - for (idx, placeholder) in placeholders.iter().enumerate() { - let separator = if idx == 0 && !source.contains('?') { '?' } else { '&' }; - source.push(separator); - source.push_str(COLLECTOR_PREFIX); - source.push_str(&idx.to_string()); - source.push('='); - source.push_str(placeholder); - } - - source -} - -fn append_siptv_template(source_url: &str) -> String { - if source_url.contains('?') { - format!("&{}", SIPTV_APPEND_TEMPLATE.trim_start_matches('?')) - } else { - SIPTV_APPEND_TEMPLATE.to_string() - } -} - -fn derive_xc_template(source_url: &str) -> Option { - let parsed = Url::parse(source_url).ok()?; - let mut segments = parsed - .path_segments() - .map(|it| it.map(ToString::to_string).collect::>())?; - if segments.len() < 3 { - return None; - } - - let file_name = segments.pop()?; - let password = segments.pop()?; - let username = segments.pop()?; - if segments.last().is_some_and(|last| last == "live") { - let _ = segments.pop(); - } - segments.push("timeshift".to_string()); - segments.push(username); - segments.push(password); - segments.push("{duration:60}".to_string()); - segments.push(XC_START_TEMPLATE.to_string()); - segments.push(file_name); - - let mut rebuilt = parsed; - let new_path = format!("/{}", segments.join("/")); - rebuilt.set_path(&new_path); - Some(rebuilt.into()) -} - -fn derive_flussonic_template(source_url: &str) -> Option { - let mut parsed = Url::parse(source_url).ok()?; - let mut segments = parsed - .path_segments() - .map(|it| it.map(ToString::to_string).collect::>())?; - let file_name = segments.pop()?; - segments.push(format!("{FLUSSONIC_TEMPLATE}{}", extract_suffix_from_filename(&file_name))); - parsed.set_path(&format!("/{}", segments.join("/"))); - Some(parsed.into()) -} - -fn append_query_template(source_url: &str, source_template: &str) -> Option { - let mut parsed = Url::parse(source_url).ok()?; - let append = source_template.strip_prefix('?').unwrap_or(source_template); - let mut merged = parsed.query().map(ToString::to_string).unwrap_or_default(); - if !merged.is_empty() && !append.is_empty() { - merged.push('&'); - } - merged.push_str(append); - parsed.set_query((!merged.is_empty()).then_some(merged.as_str())); - Some(parsed.into()) -} - -fn extract_suffix_from_filename(file_name: &str) -> &str { - file_name - .char_indices() - .find_map(|(idx, ch)| (ch == '.').then_some(&file_name[idx..])) - .unwrap_or_default() -} - -fn is_append_like_query_source(mode: &str, source: &str) -> bool { - mode_alias(mode) == "append" || source.starts_with('?') -} - -fn derived_template_for_mode<'a>(source_url: &'a str, catchup: &'a CatchupProperties) -> Option> { - let mode = catchup.mode.as_deref().unwrap_or_default(); - if let Some(source) = catchup.source.as_deref().filter(|source| !source.is_empty()) { - return Some(if is_append_like_query_source(mode, source) { - append_query_template(source_url, source).map(Cow::Owned)? - } else { - Cow::Borrowed(source) - }); - } - - match mode_alias(mode) { - "shift" => Some(Cow::Owned(format!("{source_url}{}", append_siptv_template(source_url)))), - "xc" => derive_xc_template(source_url).map(Cow::Owned), - "fs" => derive_flussonic_template(source_url).map(Cow::Owned), - "vod" => Some(Cow::Borrowed("{catchup-id}")), - _ => None, - } -} - -fn resolve_collectors(raw_query: Option<&str>) -> Result, TuliproxError> { - let mut collectors = Vec::new(); - let Some(raw_query) = raw_query else { - return Ok(collectors); - }; - - for (key, value) in url::form_urlencoded::parse(raw_query.as_bytes()) { - if key == M3U_CATCHUP_MARKER { - continue; - } - let Some(idx) = key - .strip_prefix(COLLECTOR_PREFIX) - .and_then(|suffix| suffix.parse::().ok()) else { - return Err(TuliproxError::Crypto(format!("Unsupported catchup query parameter '{key}'"))); - }; - collectors.push((idx, value.into_owned())); - } - collectors.sort_by_key(|(idx, _)| *idx); - Ok(collectors) -} - -fn render_template(segments: &[TemplateSegment], collectors: &[(usize, String)]) -> Result { - let mut collector_idx = 0usize; - let mut output = String::new(); - - for segment in segments { - match segment { - TemplateSegment::Literal(literal) => output.push_str(literal), - TemplateSegment::Placeholder(_) => { - let Some((actual_idx, value)) = collectors.get(collector_idx) else { - return Err(TuliproxError::Crypto("Missing catchup collector value".to_string())); - }; - if *actual_idx != collector_idx { - return Err(TuliproxError::Crypto("Catchup collector sequence is sparse".to_string())); - } - output.push_str(value); - collector_idx += 1; - } - } - } - - if collector_idx != collectors.len() { - return Err(TuliproxError::Crypto("Unexpected extra catchup collector values".to_string())); - } - - Ok(output) -} - -pub fn build_m3u_catchup_rewrite( - secret: &[u8; 16], - base_url: &str, - username: &str, - target_id: u16, - virtual_id: u32, - source_url: &str, - catchup: &CatchupProperties, -) -> Result, TuliproxError> { - let Some(template) = derived_template_for_mode(source_url, catchup) else { - return Ok(None); - }; - let token = encode_m3u_catchup_token( - secret, - &M3uCatchupToken { - username: username.to_string(), - target_id, - virtual_id, - }, - )?; - let segments = parse_template(template.as_ref()); - let placeholders = collect_placeholders(&segments); - let append_mode = catchup - .source - .as_deref() - .filter(|source| !source.is_empty()) - .is_some_and(|source| is_append_like_query_source(catchup.mode.as_deref().unwrap_or_default(), source)); - - let source = build_local_source(base_url, "", &token, &placeholders, append_mode); - let mode = if append_mode { "append" } else { "default" }; - Ok(Some(M3uCatchupRewrite { mode: mode.intern(), source: source.intern() })) -} - -pub fn has_m3u_catchup_marker(raw_query: Option<&str>) -> bool { - raw_query.is_some_and(|query| { - url::form_urlencoded::parse(query.as_bytes()).any(|(key, _)| key == M3U_CATCHUP_MARKER) - }) -} - -pub fn resolve_m3u_catchup_url( - source_url: &str, - catchup: &CatchupProperties, - raw_query: Option<&str>, -) -> Result, TuliproxError> { - let Some(template) = derived_template_for_mode(source_url, catchup) else { - return Ok(None); - }; - let segments = parse_template(template.as_ref()); - let placeholders = collect_placeholders(&segments); - let collectors = resolve_collectors(raw_query)?; - if collectors.len() != placeholders.len() { - return Err(TuliproxError::Crypto(format!( - "Catchup collector mismatch: expected {}, got {}", - placeholders.len(), - collectors.len() - ))); - } - - let url = render_template(&segments, &collectors)?; - let mut discriminator = url::form_urlencoded::Serializer::new(String::new()); - discriminator.append_pair("mode", catchup.mode.as_deref().unwrap_or("default")); - for (idx, value) in collectors { - discriminator.append_pair(&format!("{COLLECTOR_PREFIX}{idx}"), &value); - } - let discriminator = short_hash(&discriminator.finish()); - Ok(Some(ResolvedM3uCatchup { url, discriminator })) -} - -#[cfg(test)] -mod tests { - use super::{ - build_m3u_catchup_rewrite, has_m3u_catchup_marker, resolve_m3u_catchup_url, M3U_CATCHUP_MARKER, - }; - use shared::{model::CatchupProperties, utils::Internable}; - - #[test] - fn explicit_append_rewrite_uses_live_route_marker() { - let rewrite = build_m3u_catchup_rewrite( - &[7u8; 16], - "http://proxy.example", - "alice", - 7, - 42, - "http://provider.example/live/42.m3u8", - &CatchupProperties { - mode: Some("append".intern()), - source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), - ..CatchupProperties::default() - }, - ) - .unwrap() - .unwrap(); - - assert_eq!(rewrite.mode.as_ref(), "append"); - assert!(rewrite.source.starts_with("?tuliprox-catchup=")); - assert!(rewrite.source.contains("&v0=${offset}")); - assert!(rewrite.source.contains("&v1=${timestamp}")); - } - - #[test] - fn shift_rewrite_uses_direct_catchup_route() { - let rewrite = build_m3u_catchup_rewrite( - &[7u8; 16], - "http://proxy.example", - "alice", - 7, - 42, - "http://provider.example/live/42.ts", - &CatchupProperties { - mode: Some("shift".intern()), - ..CatchupProperties::default() - }, - ) - .unwrap() - .unwrap(); - - assert_eq!(rewrite.mode.as_ref(), "default"); - assert!(rewrite.source.starts_with("http://proxy.example/m3u-catchup/")); - assert!(rewrite.source.contains("?v0={utc}")); - assert!(rewrite.source.contains("&v1={lutc}")); - } - - #[test] - fn resolve_explicit_append_template_roundtrips_collectors() { - let resolved = resolve_m3u_catchup_url( - "http://provider.example/live/42.m3u8", - &CatchupProperties { - mode: Some("append".intern()), - source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), - ..CatchupProperties::default() - }, - Some("tuliprox-catchup=abc&v0=120&v1=1717200000"), - ) - .unwrap() - .unwrap(); - - assert_eq!( - resolved.url, - "http://provider.example/live/42.m3u8?offset=-120&utcstart=1717200000" - ); - assert!(!resolved.discriminator.is_empty()); - } - - #[test] - fn resolve_explicit_append_template_merges_into_existing_query() { - let resolved = resolve_m3u_catchup_url( - "http://provider.example/live/42.m3u8?token=abc", - &CatchupProperties { - mode: Some("append".intern()), - source: Some("?offset=-${offset}&utcstart=${timestamp}".intern()), - ..CatchupProperties::default() - }, - Some("tuliprox-catchup=abc&v0=120&v1=1717200000"), - ) - .unwrap() - .unwrap(); - - assert_eq!( - resolved.url, - "http://provider.example/live/42.m3u8?token=abc&offset=-120&utcstart=1717200000" - ); - } - - #[test] - fn resolve_default_mode_query_template_keeps_base_stream_url() { - let resolved = resolve_m3u_catchup_url( - "http://provider.example/live/42.ts", - &CatchupProperties { - mode: Some("default".intern()), - source: Some("?playseek=${timestamp}&duration=${duration}".intern()), - ..CatchupProperties::default() - }, - Some("v0=1717200000&v1=120"), - ) - .unwrap() - .unwrap(); - - assert_eq!( - resolved.url, - "http://provider.example/live/42.ts?playseek=1717200000&duration=120" - ); - } - - #[test] - fn default_mode_query_template_rewrite_uses_append_route_marker() { - let rewrite = build_m3u_catchup_rewrite( - &[7u8; 16], - "http://proxy.example", - "alice", - 7, - 42, - "http://provider.example/live/42.ts", - &CatchupProperties { - mode: Some("default".intern()), - source: Some("?playseek=${timestamp}&duration=${duration}".intern()), - ..CatchupProperties::default() - }, - ) - .unwrap() - .unwrap(); - - assert_eq!(rewrite.mode.as_ref(), "append"); - assert!(rewrite.source.starts_with("?tuliprox-catchup=")); - assert!(rewrite.source.contains("&v0=${timestamp}")); - assert!(rewrite.source.contains("&v1=${duration}")); - } - - #[test] - fn resolve_shift_template_roundtrips_collectors() { - let resolved = resolve_m3u_catchup_url( - "http://provider.example/live/42.ts", - &CatchupProperties { - mode: Some("shift".intern()), - ..CatchupProperties::default() - }, - Some("v0=20240101120000&v1=20240101130000"), - ) - .unwrap() - .unwrap(); - - assert_eq!( - resolved.url, - "http://provider.example/live/42.ts?utc=20240101120000&lutc=20240101130000" - ); - } - - #[test] - fn discriminator_changes_when_collectors_change() { - let first = resolve_m3u_catchup_url( - "http://provider.example/live/42.ts", - &CatchupProperties { - mode: Some("shift".intern()), - ..CatchupProperties::default() - }, - Some("v0=20240101120000&v1=20240101130000"), - ) - .unwrap() - .unwrap(); - let second = resolve_m3u_catchup_url( - "http://provider.example/live/42.ts", - &CatchupProperties { - mode: Some("shift".intern()), - ..CatchupProperties::default() - }, - Some("v0=20240101140000&v1=20240101150000"), - ) - .unwrap() - .unwrap(); - - assert_ne!(first.discriminator, second.discriminator); - } - - #[test] - fn catchup_marker_detection_is_explicit() { - assert!(has_m3u_catchup_marker(Some(&format!("{M3U_CATCHUP_MARKER}=abc&v0=1")))); - assert!(!has_m3u_catchup_marker(Some("v0=1"))); - assert!(!has_m3u_catchup_marker(None)); - } -} diff --git a/backend/src/utils/mod.rs b/backend/src/utils/mod.rs index 1560efafa..0aed9d462 100644 --- a/backend/src/utils/mod.rs +++ b/backend/src/utils/mod.rs @@ -8,8 +8,6 @@ mod logging; mod trakt; mod json_utils; mod provider_resolve_token; -mod m3u_catchup; -mod m3u_catchup_token; mod binary_utils; mod telegram; pub(crate) mod geoip; @@ -86,8 +84,6 @@ pub use self::crypto_utils::*; pub use self::file::*; pub use self::json_utils::*; pub use self::network::*; -pub use self::m3u_catchup::*; -pub use self::m3u_catchup_token::*; pub use self::provider_resolve_token::*; pub use self::step_measure::*; pub use self::sys_utils::*; diff --git a/backend/src/utils/network/mod.rs b/backend/src/utils/network/mod.rs index e9bcb57a3..fc97b33f2 100644 --- a/backend/src/utils/network/mod.rs +++ b/backend/src/utils/network/mod.rs @@ -4,10 +4,7 @@ use tokio::io::AsyncRead; pub(crate) mod content_coding; pub mod epg; pub mod ip_checker; -pub mod m3u; pub mod request; -pub mod stalker; -pub mod xtream; /// Type-erased asynchronous reader used by the existing network download stack. pub type DynReader = Pin>; diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index f0ac1c8f3..226636849 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -1819,7 +1819,9 @@ pub fn get_request_headers( } // 3. Finally, if no User-Agent was provided by config OR client, use the default. - if !has_user_agent { + if !has_user_agent + && !disabled_headers.is_some_and(|disabled| disabled.should_remove(axum::http::header::USER_AGENT.as_str())) + { let config_ua = default_user_agent .and_then(|ua| { let trimmed = ua.trim(); @@ -1836,6 +1838,23 @@ pub fn get_request_headers( headers } +pub fn overlay_upstream_user_agent( + headers: &mut HeaderMap, + upstream_user_agent: Option<&str>, + disabled_headers: Option<&ReverseProxyDisabledHeaderConfig>, +) { + if disabled_headers.is_some_and(|disabled| disabled.should_remove(axum::http::header::USER_AGENT.as_str())) { + return; + } + if let Some(value) = upstream_user_agent + .map(str::trim) + .filter(|value| !value.is_empty()) + .and_then(|value| HeaderValue::from_str(value).ok()) + { + headers.insert(axum::http::header::USER_AGENT, value); + } +} + // read local file content and return it as a string. // Gzipped file content is supported. pub async fn get_local_file_content(file_path: &Path) -> Result { @@ -3018,7 +3037,7 @@ mod tests { use crate::{ model::{ AppConfig, Config, ConfigInput, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, - ReverseProxyConfig, SourcesConfig, + ReverseProxyConfig, ReverseProxyDisabledHeaderConfig, SourcesConfig, }, utils::{ content_coding::{ContentCoding, ContentCodingError, ContentDecodingIoError, OutboundContentCodingPolicy}, @@ -3176,7 +3195,7 @@ mod tests { #[test] fn test_get_request_headers_prioritization() { - use super::get_request_headers; + use super::{get_request_headers, overlay_upstream_user_agent}; use axum::http::header::USER_AGENT; // Case 1: No headers provided -> Default UA @@ -3208,6 +3227,30 @@ mod tests { let headers = get_request_headers(Some(&config_headers), Some(&client_headers), None, Some("Config-Default-UA")); assert_eq!(headers.get("X-Test").unwrap(), "From-Config"); + + let mut headers = get_request_headers( + Some(&config_headers), + Some(&client_headers), + None, + Some("Config-Default-UA"), + ); + overlay_upstream_user_agent(&mut headers, Some("Channel-UA"), None); + assert_eq!(headers.get(USER_AGENT).unwrap(), "Channel-UA"); + + let disabled = ReverseProxyDisabledHeaderConfig { + referer_header: false, + x_header: false, + cloudflare_header: false, + custom_header: vec!["User-Agent".to_string()], + }; + let mut headers = get_request_headers( + Some(&config_headers), + Some(&client_headers), + Some(&disabled), + Some("Config-Default-UA"), + ); + overlay_upstream_user_agent(&mut headers, Some("Blocked-Channel-UA"), Some(&disabled)); + assert!(!headers.contains_key(USER_AGENT)); } #[test] diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index a6bf57a8f..f9f237e41 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -1632,8 +1632,7 @@ output: #### M3U Catchup & Archive Support -Tuliprox preserves standard M3U catchup/archive attributes from provider playlists. -The following attributes are recognized and preserved during M3U import and export: +Tuliprox preserves the following catchup and archive attributes when it imports and exports M3U playlists: * `catchup` * `catchup-days` @@ -1641,19 +1640,33 @@ The following attributes are recognized and preserved during M3U import and expo * `catchup-time` * `catchup-correction` * `catchup-type` -* Any unknown `catchup-*` attributes +* unknown attributes whose names start with `catchup-` -**Proxy Behavior:** +When an output keeps direct provider URLs, the catchup metadata is written unchanged. For reverse-proxied outputs, +Tuliprox replaces supported provider templates with authenticated local URLs. The provider URL and its credentials are +not included in the generated catchup URL. -* **Direct Output:** When Tuliprox outputs streams directly (e.g., when not using proxy rewrite), it preserves the original provider - metadata exactly as received. -* **Rewritten Output (Reverse Proxy):** When Tuliprox rewrites URLs to act as a reverse proxy, it generates authenticated local Tuliprox - archive URLs instead of leaking the provider's upstream archive URLs. +The template modes `default`, `append`, `shift`, `xc`, `fs`, and `vod` are supported. An unknown mode can still be +proxied when it supplies an explicit source template. Without a usable template, Tuliprox preserves the metadata but +cannot create a local catchup URL. -**Supported Modes:** -Tuliprox supports processing and proxying the following catchup modes: `default`, `append`, `shift`, `xc`, `fs`, and `vod`. -For unknown modes with an explicit source template, Tuliprox attempts to safely proxy them. For unknown modes without a source template, -Tuliprox preserves the metadata but logs a limitation that it cannot safely proxy the unknown mode locally. +Native Flussonic playback is available for both HLS and MPEG-TS: + +* HLS uses `.m3u8` archive, relative-timeshift, and absolute-timeshift paths. +* MPEG-TS uses absolute-timeshift `.ts` paths. +* Flat archive requests generated by TiviMate and nested Flussonic archive paths are accepted. +* A live item with a `timeshift` value but no catchup block is treated as Flussonic-style catchup. + +For HLS archives, Tuliprox carries the archive start parameter into same-origin child playlists when the child does not +already contain one. It does not add archive parameters to media segments, encryption keys, or initialization files. + +When API server information is available, the generated `#EXTM3U` header contains `url-tvg` and `x-tvg-url`. Both point +to the authenticated Tuliprox XMLTV endpoint for the playlist user; a provider-supplied `url-tvg` value is not forwarded. + +Tuliprox also reads a per-channel `#EXTVLCOPT:http-user-agent` directive. It writes the directive back when the output +contains the direct provider URL. For rewritten URLs, the value is kept internal and applied to the upstream HLS or +MPEG-TS request instead of being exposed in the playlist. Disabling the `User-Agent` header through the reverse-proxy +header settings takes precedence. ### 3. Type `strm` diff --git a/docs/src/features.md b/docs/src/features.md index ac9a7b2b2..e37461e5b 100644 --- a/docs/src/features.md +++ b/docs/src/features.md @@ -30,6 +30,7 @@ That makes it usable both for IPTV players and for media-server-oriented workflo ## Runtime streaming - reverse-proxy streams instead of redirecting them +- proxy M3U catchup and archive playback for template-based and native Flussonic HLS or MPEG-TS URLs - keep provider account affinity where clients need it - share live streams across users - enforce user connection limits diff --git a/frontend/src/app/components/dashboard/stream_display/helpers.rs b/frontend/src/app/components/dashboard/stream_display/helpers.rs index d7329be00..449b4ac4a 100644 --- a/frontend/src/app/components/dashboard/stream_display/helpers.rs +++ b/frontend/src/app/components/dashboard/stream_display/helpers.rs @@ -240,6 +240,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }, provider: "provider".intern(), addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080), diff --git a/frontend/src/hooks/use_server_status.rs b/frontend/src/hooks/use_server_status.rs index a7bad0594..1dcbbc7e9 100644 --- a/frontend/src/hooks/use_server_status.rs +++ b/frontend/src/hooks/use_server_status.rs @@ -104,6 +104,7 @@ fn download_task_to_stream_with_ts(download: &FileDownloadDto, ts: u64) -> Strea technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }, provider: BACKGROUND_TRANSFER_PROVIDER.intern(), addr: download_stream_addr(uid), @@ -391,6 +392,7 @@ mod tests { technical: None, epg_channel_id: None, epg_reference_ts: None, + upstream_user_agent: None, }, provider: "provider".intern(), addr: addr.parse::().unwrap_or_else(|_| unreachable!()), diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index da376557c..a9ff41cba 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -290,6 +290,7 @@ pub trait PlaylistEntry: Send + Sync { fn get_resolved_info_document(&self, options: &XtreamMappingOptions) -> Option; fn get_additional_properties(&self) -> Option<&StreamProperties>; fn get_additional_properties_mut(&mut self) -> Option<&mut StreamProperties>; + fn get_upstream_user_agent(&self) -> Option<&str>; } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -340,6 +341,8 @@ pub struct PlaylistItemHeader { /// Stable provider/origin ID captured before any target transformation. #[serde(default, with = "arc_str_serde")] pub input_stream_id: Arc, + #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] + pub upstream_user_agent: Option>, } impl Default for PlaylistItemHeader { @@ -367,6 +370,7 @@ impl Default for PlaylistItemHeader { input_name: "".intern(), source_ordinal: 0, input_stream_id: "".intern(), + upstream_user_agent: None, } } } @@ -571,6 +575,8 @@ pub struct M3uPlaylistItem { /// Stable provider/origin ID captured before any target transformation. #[serde(default, with = "arc_str_serde")] pub input_stream_id: Arc, + #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] + pub upstream_user_agent: Option>, } fn write_m3u_attr(line: &mut String, name: &str, value: &str) { let _ = write!(line, " {name}=\"{value}\""); } @@ -653,8 +659,14 @@ impl M3uPlaylistItem { } } + let _ = writeln!(&mut line, ",{}", self.title); + if let Some(user_agent) = + self.upstream_user_agent.as_deref().filter(|value| !value.is_empty() && !value.contains(['\r', '\n'])) + { + let _ = writeln!(&mut line, "#EXTVLCOPT:http-user-agent={user_agent}"); + } let url = if self.t_stream_url.is_empty() { &self.url } else { &self.t_stream_url }; - let _ = write!(&mut line, ",{}\n{}", self.title, url); + line.push_str(url); line } @@ -692,6 +704,8 @@ impl PlaylistEntry for M3uPlaylistItem { (!input_stream_id.is_empty()).then(|| Arc::clone(input_stream_id)) } + fn get_upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } + fn get_provider_id(&self) -> Option { get_provider_id(&self.provider_id, &self.url) } #[inline] fn get_category_id(&self) -> Option { None } @@ -919,6 +933,8 @@ pub struct XtreamPlaylistItem { /// Stable provider/origin ID captured before any target transformation. #[serde(default, with = "arc_str_serde")] pub input_stream_id: Arc, + #[serde(default, rename = "source_user_agent", alias = "upstream_user_agent", with = "arc_str_option_serde")] + pub upstream_user_agent: Option>, } impl XtreamPlaylistItem { @@ -982,6 +998,8 @@ impl PlaylistEntry for XtreamPlaylistItem { Some(Arc::clone(&self.input_stream_id)) } } + + fn get_upstream_user_agent(&self) -> Option<&str> { self.upstream_user_agent.as_deref() } #[inline] fn get_provider_id(&self) -> Option { Some(self.provider_id) } #[inline] @@ -1239,6 +1257,7 @@ impl From<&PlaylistItem> for XtreamPlaylistItem { channel_no: header.chno, source_ordinal: header.source_ordinal, input_stream_id, + upstream_user_agent: header.upstream_user_agent.clone(), } } } @@ -1277,6 +1296,7 @@ impl From<&PlaylistItem> for M3uPlaylistItem { source_ordinal: header.source_ordinal, additional_properties: header.additional_properties.clone(), input_stream_id, + upstream_user_agent: header.upstream_user_agent.clone(), } } } @@ -1345,6 +1365,7 @@ impl From<&XtreamPlaylistItem> for PlaylistItem { additional_properties: item.additional_properties.clone(), source_ordinal: item.source_ordinal, input_stream_id: input_stream_id.unwrap_or_else(|| "".intern()), + upstream_user_agent: item.upstream_user_agent.clone(), }; PlaylistItem { header } @@ -1376,6 +1397,7 @@ impl From<&M3uPlaylistItem> for PlaylistItem { additional_properties: item.additional_properties.clone(), source_ordinal: item.source_ordinal, input_stream_id: item.get_input_stream_id().unwrap_or_else(|| "".intern()), + upstream_user_agent: item.upstream_user_agent.clone(), }; PlaylistItem { header } @@ -1445,6 +1467,7 @@ impl PlaylistItem { category_id: item.category_id, source_ordinal: 0, input_stream_id: stream_id_str, + upstream_user_agent: None, }; PlaylistItem { header } @@ -1458,6 +1481,8 @@ impl PlaylistEntry for PlaylistItem { #[inline] fn get_input_stream_id(&self) -> Option> { self.header.get_input_stream_id() } + fn get_upstream_user_agent(&self) -> Option<&str> { self.header.upstream_user_agent.as_deref() } + fn get_provider_id(&self) -> Option { let header = &self.header; get_provider_id(&header.id, &header.url) @@ -1876,6 +1901,7 @@ mod tests { virtual_id: 0, provider_id: "prov1".intern(), input_stream_id: "prov1".intern(), + upstream_user_agent: None, name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -1910,6 +1936,7 @@ mod tests { virtual_id: 0, provider_id: "prov1".intern(), input_stream_id: "prov1".intern(), + upstream_user_agent: None, name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -1942,6 +1969,7 @@ mod tests { virtual_id: 0, provider_id: "prov1".intern(), input_stream_id: "prov1".intern(), + upstream_user_agent: None, name: "Test Channel".intern(), chno: 42, logo: "".intern(), @@ -1974,6 +2002,7 @@ mod tests { virtual_id: 0, provider_id: "prov1".intern(), input_stream_id: "prov1".intern(), + upstream_user_agent: None, name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -2006,6 +2035,7 @@ mod tests { virtual_id: 0, provider_id: "prov1".intern(), input_stream_id: "prov1".intern(), + upstream_user_agent: None, name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -2054,6 +2084,7 @@ mod tests { virtual_id: 0, provider_id: "prov1".intern(), input_stream_id: "prov1".intern(), + upstream_user_agent: None, name: "Test Channel".intern(), chno: 0, logo: "".intern(), @@ -2088,4 +2119,14 @@ mod tests { assert!(output.contains(r#"catchup-source="http://proxy.example/m3u-catchup/token?v0={utc}""#)); assert!(!output.contains(r#"catchup-source="?offset=-${offset}""#)); } + + #[test] + fn m3u_to_m3u_emits_only_configured_upstream_user_agent() { + let mut item = M3uPlaylistItem::from(&PlaylistItem { header: PlaylistItemHeader::default() }); + item.upstream_user_agent = Some("Provider-UA".intern()); + + assert!(item.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=Provider-UA\n")); + item.upstream_user_agent = None; + assert!(!item.to_m3u(None, false).contains("#EXTVLCOPT:http-user-agent=")); + } } diff --git a/shared/src/model/stream_info.rs b/shared/src/model/stream_info.rs index 49c512137..fd0102c5d 100644 --- a/shared/src/model/stream_info.rs +++ b/shared/src/model/stream_info.rs @@ -63,6 +63,8 @@ pub struct StreamChannel { pub epg_channel_id: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub epg_reference_ts: Option, + #[serde(skip)] + pub upstream_user_agent: Option>, } impl StreamChannel { @@ -107,6 +109,7 @@ impl XtreamPlaylistItem { technical: stream_technical_from_properties(self.additional_properties.as_ref(), self.url.as_ref()), epg_channel_id: self.epg_channel_id.clone(), epg_reference_ts: None, + upstream_user_agent: self.upstream_user_agent.clone(), } } } @@ -130,6 +133,7 @@ impl M3uPlaylistItem { technical: stream_technical_from_properties(self.additional_properties.as_ref(), self.url.as_ref()), epg_channel_id: self.epg_channel_id.clone(), epg_reference_ts: None, + upstream_user_agent: self.upstream_user_agent.clone(), } } } @@ -384,6 +388,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "1".intern(), + upstream_user_agent: None, }; let stream_channel = create_stream_channel_with_type(1, &playlist_item, PlaylistItemType::Video); @@ -414,6 +419,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "1".intern(), + upstream_user_agent: None, }; let stream_channel = playlist_item.to_stream_channel(1); @@ -421,6 +427,37 @@ mod tests { assert_eq!(stream_channel.input_name.as_ref(), "provider-input"); } + #[test] + fn stream_channel_keeps_upstream_user_agent_internal() { + let mut playlist_item = XtreamPlaylistItem { + virtual_id: 1, + provider_id: 1, + name: "Example".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "Live".intern(), + title: "Example".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://provider.example/live/1.ts".intern(), + epg_channel_id: None, + xtream_cluster: XtreamCluster::Live, + additional_properties: None, + item_type: PlaylistItemType::Live, + category_id: 0, + input_name: "provider-input".intern(), + channel_no: 0, + source_ordinal: 0, + input_stream_id: "1".intern(), + upstream_user_agent: Some("secret-provider-agent".intern()), + }; + let stream_channel = playlist_item.to_stream_channel(1); + playlist_item.upstream_user_agent = None; + + assert_eq!(stream_channel.upstream_user_agent.as_deref(), Some("secret-provider-agent")); + assert!(!serde_json::to_string(&stream_channel).is_ok_and(|json| json.contains("secret-provider-agent"))); + } + #[test] fn m3u_to_stream_channel_with_epg_reference_ts_propagates_value() { let pli = M3uPlaylistItem { @@ -447,6 +484,7 @@ mod tests { source_ordinal: 0, additional_properties: None, input_stream_id: "42".intern(), + upstream_user_agent: None, }; let channel = pli.to_stream_channel(7).with_epg_reference_ts(Some(1_700_000_000)); @@ -483,6 +521,7 @@ mod tests { source_ordinal: 0, additional_properties: None, input_stream_id: "42".intern(), + upstream_user_agent: None, }; // to_stream_channel() defaults to None, and with_epg_reference_ts(None) diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index 9f5730a19..4f3c8b289 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -47,6 +47,22 @@ impl CatchupProperties { && self.catchup_type.is_none() && self.extra_attributes.is_empty() } + + pub fn native_flussonic_player_mode(&self) -> Option<&'static str> { + let mode = self + .mode + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .or_else(|| self.catchup_type.as_deref().map(str::trim).filter(|value| !value.is_empty()))?; + if mode.eq_ignore_ascii_case("flussonic-ts") { + Some("flussonic-ts") + } else if ["fs", "flussonic", "flussonic-hls"].iter().any(|alias| mode.eq_ignore_ascii_case(alias)) { + Some("flussonic") + } else { + None + } + } } fn format_episode_code(season: u32, episode: u32) -> Option { @@ -1052,6 +1068,7 @@ mod tests { channel_no: 0, source_ordinal: 0, input_stream_id: "1001".into(), + upstream_user_agent: None, } } @@ -1145,4 +1162,25 @@ mod tests { assert!(props.has_details()); } + + #[test] + fn native_flussonic_mode_is_canonical_and_respects_mode_precedence() { + for (value, expected) in [ + ("fs", Some("flussonic")), + ("FLUSSONIC", Some("flussonic")), + ("flussonic-hls", Some("flussonic")), + ("Flussonic-TS", Some("flussonic-ts")), + ("append", None), + ] { + let catchup = CatchupProperties { mode: Some(value.into()), ..CatchupProperties::default() }; + assert_eq!(catchup.native_flussonic_player_mode(), expected); + } + + let conflicting = CatchupProperties { + mode: Some("append".into()), + catchup_type: Some("flussonic".into()), + ..CatchupProperties::default() + }; + assert_eq!(conflicting.native_flussonic_player_mode(), None); + } }