From 90409406746391f7ac82d54c14d42e2d68342f38 Mon Sep 17 00:00:00 2001 From: knylbyte <40831653+knylbyte@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:53:33 +0200 Subject: [PATCH] feat: ics to xmltv exporter (#784) feat: ics to xmltv exporter --- CHANGELOG.md | 4 + Cargo.lock | 1 + backend/src/api/endpoints/v1_api_playlist.rs | 780 ++++++++++++------ backend/src/api/endpoints/xmltv_api.rs | 77 +- backend/src/api/endpoints/xtream_api.rs | 173 +++- backend/src/api/model/app_state.rs | 73 ++ backend/src/api/sys_usage.rs | 17 +- backend/src/model/config/epg.rs | 155 +++- backend/src/model/config/reverse_proxy.rs | 6 +- backend/src/model/xmltv.rs | 28 +- backend/src/processing/parser/ics/dummy.rs | 458 ++++++++++ backend/src/processing/parser/ics/event.rs | 603 ++++++++++++++ backend/src/processing/parser/ics/mod.rs | 274 ++++++ backend/src/processing/parser/ics/time.rs | 189 +++++ backend/src/processing/parser/mod.rs | 3 +- backend/src/processing/parser/xmltv.rs | 563 ++++++++++--- backend/src/processing/parser/xtream.rs | 16 + backend/src/processing/processor/epg.rs | 180 +++- backend/src/repository/epg_repository.rs | 104 ++- backend/src/utils/network/epg.rs | 329 +++++++- backend/src/utils/network/request.rs | 724 ++++++++++++++-- docs/src/configuration/source.md | 393 ++++++++- .../source_editor/epg_source_item_form.rs | 10 +- shared/Cargo.toml | 1 + shared/src/defaults/epg.rs | 37 + shared/src/model/config/epg.rs | 594 ++++++++++++- shared/src/model/config/input.rs | 4 +- 27 files changed, 5211 insertions(+), 585 deletions(-) create mode 100644 backend/src/processing/parser/ics/dummy.rs create mode 100644 backend/src/processing/parser/ics/event.rs create mode 100644 backend/src/processing/parser/ics/mod.rs create mode 100644 backend/src/processing/parser/ics/time.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ad448123c..c8165615d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ ## 🌟 New Features +- **ICS Calendar EPG Sources**: Import iCalendar (`.ics`) events as XMLTV EPG data with M3U and Xtream channel + assignment, Smart Match support, configurable four-hour dummy gap filling, bounded atomic cache downloads, and + aggregated warnings for recurring events that are detected but not expanded yet. + - **Trakt Charts**: Xtream Trakt integration can now build virtual categories from public Trakt charts via `trakt.charts[]`. - MVP supports `movies/shows` with `trending` and `popular`. - User-owned Trakt lists remain configured separately under `trakt.lists[]`. diff --git a/Cargo.lock b/Cargo.lock index 6544f51a0..c3021c505 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3998,6 +3998,7 @@ dependencies = [ "brotli", "bytes", "chrono", + "chrono-tz", "ciborium", "dashmap", "deunicode", diff --git a/backend/src/api/endpoints/v1_api_playlist.rs b/backend/src/api/endpoints/v1_api_playlist.rs index 2708c8bfe..d170dc8f9 100644 --- a/backend/src/api/endpoints/v1_api_playlist.rs +++ b/backend/src/api/endpoints/v1_api_playlist.rs @@ -1,28 +1,45 @@ -use crate::api::api_utils::resource_response; -use crate::{api::{ - api_utils::{create_api_proxy_user, json_or_bin_response, try_option_bad_request, try_result_bad_request, try_unwrap_body}, - endpoints::{ - api_playlist_utils::{get_playlist_for_custom_provider, get_playlist_for_input, get_playlist_for_target}, - extract_accept_header::ExtractAcceptHeader, - m3u_api::m3u_api_stream_loaded, - xmltv_api::{rewrite_epg_channel_resource_url, serve_epg_web_ui, stream_epg_api}, - xtream_api::{xtream_get_stream_info_response, xtream_player_api_stream_with_token, ApiStreamContext, ApiStreamRequest}, +use crate::{ + api::{ + api_utils::{ + create_api_proxy_user, json_or_bin_response, resource_response, try_option_bad_request, + try_result_bad_request, try_unwrap_body, + }, + endpoints::{ + api_playlist_utils::{get_playlist_for_custom_provider, get_playlist_for_input, get_playlist_for_target}, + extract_accept_header::ExtractAcceptHeader, + m3u_api::m3u_api_stream_loaded, + xmltv_api::{rewrite_epg_channel_resource_url, serve_epg_web_ui, stream_epg_api}, + xtream_api::{ + xtream_get_stream_info_response, xtream_player_api_stream_with_token, ApiStreamContext, + ApiStreamRequest, + }, + }, + model::AppState, }, - model::AppState, -}, auth::{create_access_token, permission_layer, verify_access_token}, model::{parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, AppConfig, ConfigInput, ConfigInputFlags, ConfigInputOptions, InputSource}, processing::parser::xmltv::merge_epg_channels_by_priority, repository::{m3u_get_item_for_stream_id, xtream_get_item_for_stream_id}, utils::{epg::get_input_raw_epg_file_path, file_exists_async, xtream}}; + auth::{create_access_token, permission_layer, verify_access_token}, + model::{ + parse_xmltv_for_web_ui_from_file, parse_xmltv_for_web_ui_from_url, AppConfig, ConfigInput, ConfigInputFlags, + ConfigInputOptions, EpgSource, EpgSourceType, IcsDummyPolicy, InputSource, + }, + processing::parser::{ + ics::parse_ics_file_to_channel, + xmltv::{merge_epg_channels_by_priority_with_dummy_policies, EpgDummyPolicySource}, + }, + repository::{m3u_get_item_for_stream_id, xtream_get_item_for_stream_id}, + utils::{epg::get_input_raw_epg_file_path, file_exists_async, request, xtream}, +}; use axum::{response::IntoResponse, Router}; use log::{debug, error}; use serde_json::json; -use shared::utils::deobfuscate_text; use shared::{ + error::TuliproxError, model::{ - permission::Permission, EpgChannel, OperationRunAccepted, - InputType, PlaylistEpgRequest, PlaylistRequest, PlaylistUrlResolveRequest, ProxyType, TargetType, UiPlaylistItem, - XtreamCluster, + permission::Permission, EpgChannel, InputType, OperationRunAccepted, PlaylistEpgRequest, PlaylistRequest, + PlaylistUrlResolveRequest, ProxyType, TargetType, UiPlaylistItem, XtreamCluster, }, - utils::{concat_path_leading_slash, sanitize_sensitive_info, Internable}, + utils::{concat_path_leading_slash, deobfuscate_text, sanitize_sensitive_info, Internable}, }; -use std::{str::FromStr, sync::Arc}; +use std::{path::Path, str::FromStr, sync::Arc}; use url::Url; fn create_config_input_for_m3u(url: &str) -> ConfigInput { @@ -91,9 +108,7 @@ fn resolve_provider_url_for_request(app_config: &AppConfig, playlist_request: &P .get_target_by_id(*target_id) .and_then(|target| app_config.get_inputs_for_target(&target.name)) .and_then(|inputs| { - let mut matches = inputs - .into_iter() - .filter(|input| input.get_resolve_provider(url).is_some()); + let mut matches = inputs.into_iter().filter(|input| input.get_resolve_provider(url).is_some()); let first = matches.next()?; if matches.next().is_some() { return None; @@ -120,84 +135,167 @@ fn build_playlist_webplayer_url( ) } +#[cfg(test)] fn merge_epg_channels(mut channels_by_source: Vec<(i16, Vec)>) -> Vec { channels_by_source.sort_by_key(|(priority, _)| *priority); - merge_epg_channels_by_priority(channels_by_source) + merge_epg_channels_by_priority_with_dummy_policies(channels_by_source, Vec::new()) } -async fn load_epg_channels_for_input( +async fn load_xmltv_epg_source_channels( app_state: &Arc, - input: &ConfigInput, -) -> Result>, shared::error::TuliproxError> { - let Some(epg_config) = input.epg.as_ref() else { - return Ok(None); - }; - - let storage_dir = app_state.app_config.config.load().storage_dir.clone(); - let mut channels_by_source = Vec::new(); - let mut failed_sources = 0usize; - - for epg_source in &epg_config.sources { - let resolved_url = resolve_provider_url_with_input(input, &epg_source.url); - let raw_epg_path = match get_input_raw_epg_file_path(&resolved_url, input, &storage_dir).await { - Ok(path) => path, - Err(err) => { - debug!( - "Skipping EPG source {}: {}", - sanitize_sensitive_info(resolved_url.as_str()), - sanitize_sensitive_info(&err.to_string()) - ); - failed_sources += 1; - continue; - } - }; - - let source_channels = if file_exists_async(&raw_epg_path).await { - match parse_xmltv_for_web_ui_from_file(&raw_epg_path).await { - Ok(ch) => ch, + raw_epg_path: &Path, + resolved_url: &str, +) -> Result, TuliproxError> { + { + let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; + if file_exists_async(raw_epg_path).await { + match parse_xmltv_for_web_ui_from_file(raw_epg_path).await { + Ok(channels) => return Ok(channels), Err(file_err) => { debug!( "EPG file parse failed {}, trying upstream: {file_err}", sanitize_sensitive_info(raw_epg_path.to_str().unwrap_or_default()) ); - match parse_xmltv_for_web_ui_from_url(app_state, &resolved_url).await { - Ok(ch) => ch, - Err(url_err) => { - debug!("EPG url also failed {}: {}", - sanitize_sensitive_info(resolved_url.as_str()), - sanitize_sensitive_info(&url_err.to_string())); - failed_sources += 1; - continue; - } - } } } - } else { - match parse_xmltv_for_web_ui_from_url(app_state, &resolved_url).await { - Ok(ch) => ch, - Err(err) => { - debug!("Skipping EPG url {}: {}", - sanitize_sensitive_info(resolved_url.as_str()), - sanitize_sensitive_info(&err.to_string())); - failed_sources += 1; - continue; + } + } + + parse_xmltv_for_web_ui_from_url(app_state, resolved_url).await +} + +async fn load_ics_epg_source_channels( + app_state: &Arc, + input: &ConfigInput, + epg_source: &EpgSource, + raw_epg_path: &Path, + storage_dir: &str, +) -> Result, TuliproxError> { + let ics_config = epg_source + .ics + .as_ref() + .ok_or_else(|| TuliproxError::ConfigEpg("ics configuration is required for ICS EPG sources".to_string()))?; + + let channel_id = epg_source + .channel_id + .clone() + .ok_or_else(|| TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string()))?; + + { + let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; + if file_exists_async(raw_epg_path).await { + match parse_ics_file_to_channel( + raw_epg_path, + channel_id.clone(), + epg_source.channel_title.clone(), + ics_config, + ) + .await + { + Ok(channel) => return Ok(vec![channel]), + Err(file_err) => { + debug!( + "ICS EPG file parse failed {}, redownloading source: {}", + sanitize_sensitive_info(raw_epg_path.to_str().unwrap_or_default()), + sanitize_sensitive_info(&file_err.to_string()) + ); } } + } + } + + let client = app_state.http_client.load(); + request::get_input_epg_content_as_file( + &app_state.app_config, + &client, + input, + request::InputEpgFileRequest { + headers: None, + storage_dir, + url: &epg_source.url, + persist_path: raw_epg_path, + max_bytes: Some(ics_config.max_download_bytes), + }, + ) + .await?; + + let _cache_lock = app_state.app_config.file_locks.read_lock(raw_epg_path).await; + parse_ics_file_to_channel(raw_epg_path, channel_id, epg_source.channel_title.clone(), ics_config) + .await + .map(|channel| vec![channel]) +} + +async fn load_epg_channels_for_input( + app_state: &Arc, + input: &ConfigInput, +) -> Result>, TuliproxError> { + let Some(epg_config) = input.epg.as_ref() else { + return Ok(None); + }; + + let storage_dir = app_state.app_config.config.load().storage_dir.clone(); + let mut channels_by_source = Vec::new(); + let mut dummy_policies = Vec::new(); + let mut failed_sources = 0usize; + + for (source_order, epg_source) in epg_config.sources.iter().enumerate() { + let raw_epg_path = match get_input_raw_epg_file_path(epg_source, input, &storage_dir).await { + Ok(path) => path, + Err(err) => { + debug!( + "Skipping EPG source {}: {}", + sanitize_sensitive_info(epg_source.url.as_str()), + sanitize_sensitive_info(&err.to_string()) + ); + failed_sources += 1; + continue; + } }; + + let source_result = match epg_source.source_type { + EpgSourceType::Xmltv => { + let resolved_url = resolve_provider_url_with_input(input, &epg_source.url); + load_xmltv_epg_source_channels(app_state, &raw_epg_path, &resolved_url).await + } + EpgSourceType::Ics => { + load_ics_epg_source_channels(app_state, input, epg_source, &raw_epg_path, &storage_dir).await + } + }; + + let source_channels = match source_result { + Ok(channels) => channels, + Err(err) => { + debug!( + "Skipping EPG source {}: {}", + sanitize_sensitive_info(epg_source.url.as_str()), + sanitize_sensitive_info(&err.to_string()) + ); + failed_sources += 1; + continue; + } + }; + if epg_source.source_type == EpgSourceType::Ics { + if let (Some(ics_config), Some(channel_id)) = (epg_source.ics.as_ref(), epg_source.channel_id.as_ref()) { + dummy_policies.push(EpgDummyPolicySource { + priority: epg_source.priority, + source_order, + channel_id: channel_id.clone(), + policy: IcsDummyPolicy { timezone: ics_config.timezone.clone(), config: ics_config.dummy.clone() }, + }); + } + } channels_by_source.push((epg_source.priority, source_channels)); } if channels_by_source.is_empty() { if failed_sources > 0 { - Err(shared::error::TuliproxError::Config(format!( - "All {failed_sources} EPG source(s) failed for input '{}'", - input.name - ))) + Err(TuliproxError::Config(format!("All {failed_sources} EPG source(s) failed for input '{}'", input.name))) } else { Ok(None) } } else { - Ok(Some(merge_epg_channels(channels_by_source))) + channels_by_source.sort_by_key(|(priority, _)| *priority); + Ok(Some(merge_epg_channels_by_priority_with_dummy_policies(channels_by_source, dummy_policies))) } } @@ -249,26 +347,20 @@ async fn playlist_content( cluster, accept.as_deref(), ) - .await - .into_response(), + .await + .into_response(), PlaylistRequest::Input(input_name) => get_playlist_for_input( app_state.app_config.get_input_by_name(&input_name.intern()).as_ref(), app_state, cluster, accept.as_deref(), ) - .await - .into_response(), + .await + .into_response(), PlaylistRequest::CustomXtream(xtream) => match Url::parse(&xtream.url) { Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { let input = Arc::new(create_config_input_for_xtream(&xtream.username, &xtream.password, &xtream.url)); - get_playlist_for_custom_provider( - client.as_ref(), - Some(&input), - app_state, - cluster, - accept.as_deref(), - ) + get_playlist_for_custom_provider(client.as_ref(), Some(&input), app_state, cluster, accept.as_deref()) .await .into_response() } @@ -281,13 +373,7 @@ async fn playlist_content( PlaylistRequest::CustomM3u(m3u) => match Url::parse(&m3u.url) { Ok(parsed) if parsed.scheme() == "http" || parsed.scheme() == "https" => { let input = Arc::new(create_config_input_for_m3u(&m3u.url)); - get_playlist_for_custom_provider( - client.as_ref(), - Some(&input), - app_state, - cluster, - accept.as_deref(), - ) + get_playlist_for_custom_provider(client.as_ref(), Some(&input), app_state, cluster, accept.as_deref()) .await .into_response() } @@ -336,8 +422,8 @@ async fn playlist_series_info( &virtual_id, XtreamCluster::Series, ) - .await - .into_response(); + .await + .into_response(); } } } @@ -350,11 +436,9 @@ async fn playlist_series_info( // Input/custom requests only provide provider_id, so we resolve series info from // the upstream Xtream API using provider_id. if let Some(provider_id) = provider_id { - if let Some(info_url) = xtream::get_xtream_player_api_info_url( - input.as_ref(), - XtreamCluster::Series, - provider_id, - ) { + if let Some(info_url) = + xtream::get_xtream_player_api_info_url(input.as_ref(), XtreamCluster::Series, provider_id) + { let Ok(resolved_url) = input.resolve_url(&info_url) else { return axum::http::StatusCode::NO_CONTENT.into_response(); }; @@ -380,7 +464,9 @@ async fn playlist_series_info( PlaylistRequest::CustomXtream(xtream_req) => { if let Some(provider_id) = provider_id { let input = create_config_input_for_xtream(&xtream_req.username, &xtream_req.password, &xtream_req.url); - if let Some(info_url) = xtream::get_xtream_player_api_info_url(&input, XtreamCluster::Series, provider_id) { + if let Some(info_url) = + xtream::get_xtream_player_api_info_url(&input, XtreamCluster::Series, provider_id) + { let input_source = InputSource::from(&input).with_url(info_url); if let Ok(content) = xtream::get_xtream_stream_info_content( &app_state.app_config, @@ -488,11 +574,7 @@ async fn playlist_epg( TargetType::Xtream, ) .or_else(|| { - crate::api::endpoints::xmltv_api::get_epg_path_for_target_by_type( - config, - &target, - TargetType::M3u, - ) + crate::api::endpoints::xmltv_api::get_epg_path_for_target_by_type(config, &target, TargetType::M3u) }); if let Some(epg_path) = epg_path { return serve_epg_web_ui(&app_state, accept.as_deref(), &epg_path, &target).await; @@ -504,7 +586,8 @@ async fn playlist_epg( match load_epg_channels_for_input(&app_state, input.as_ref()).await { Ok(Some(epg)) => { let config = app_state.app_config.config.load(); - let web_ui_path = config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); + let web_ui_path = + config.web_ui.as_ref().and_then(|w| w.path.as_ref()).map_or("", String::as_str); let resource_url = concat_path_leading_slash(web_ui_path, "api/v1/playlist/resource"); let encrypt_secret = app_state.get_encrypt_secret(); let epg = epg @@ -515,7 +598,11 @@ async fn playlist_epg( } Ok(None) => return axum::http::StatusCode::NO_CONTENT.into_response(), Err(err) => { - error!("Failed to load input EPG for '{}': {}", input.name, sanitize_sensitive_info(&err.to_string())); + error!( + "Failed to load input EPG for '{}': {}", + input.name, + sanitize_sensitive_info(&err.to_string()) + ); return ( axum::http::StatusCode::INTERNAL_SERVER_ERROR, axum::Json(serde_json::json!({"error": "Failed to load EPG"})), @@ -579,13 +666,7 @@ async fn playlist_resolve_url( ) -> impl IntoResponse + Send { match request { PlaylistUrlResolveRequest::Webplayer { target_id, virtual_id, cluster } => { - playlist_webplayer( - axum::extract::State(app_state), - target_id, - virtual_id, - cluster, - ) - .into_response() + playlist_webplayer(axum::extract::State(app_state), target_id, virtual_id, cluster).into_response() } PlaylistUrlResolveRequest::Provider { playlist_request, url } => { resolve_provider_url_for_request(&app_state.app_config, &playlist_request, &url).into_response() @@ -606,15 +687,11 @@ pub fn v1_api_playlist_register_protected(router: Router>) -> axum .route("/playlist/series/episode/{virtual_id}", axum::routing::post(playlist_episode_item)) } -pub fn v1_api_playlist_register_public( - router: Router>, -) -> axum::Router> { - router - .route("/playlist/resource/{resource}", axum::routing::get(playlist_resource)) - .route( - "/playlist/webplayer/{token}/{target_id}/{cluster}/{stream_id}", - axum::routing::get(playlist_webplayer_stream), - ) +pub fn v1_api_playlist_register_public(router: Router>) -> axum::Router> { + router.route("/playlist/resource/{resource}", axum::routing::get(playlist_resource)).route( + "/playlist/webplayer/{token}/{target_id}/{cluster}/{stream_id}", + axum::routing::get(playlist_webplayer_stream), + ) } pub fn v1_api_playlist_register_with_permissions( @@ -639,11 +716,7 @@ pub fn v1_api_playlist_register_with_permissions( .route("/epg/stream", axum::routing::post(stream_epg_api)) .layer(permission_layer!(app_state, Permission::EpgRead)); - router.nest("/playlist", - read_routes - .merge(write_routes) - .merge(epg_routes), - ) + router.nest("/playlist", read_routes.merge(write_routes).merge(epg_routes)) } async fn playlist_episode_item( @@ -675,9 +748,14 @@ mod tests { ActiveProviderManager, ActiveUserManager, AppState, ConnectionManager, EventManager, MetadataUpdateManager, PlaylistStorageState, SharedStreamManager, }, - model::{AppConfig, Config, ConfigInput, ConfigProvider, ConfigSource, ConfigTarget, SourcesConfig, StreamHistoryConfig}, - utils::epg::get_input_raw_epg_file_path, - utils::GeoIp, + model::{ + AppConfig, Config, ConfigInput, ConfigProvider, ConfigSource, ConfigTarget, SourcesConfig, + StreamHistoryConfig, + }, + utils::{ + epg::{get_input_raw_epg_file_path, get_input_raw_xmltv_file_path}, + GeoIp, + }, }; use arc_swap::{ArcSwap, ArcSwapOption}; use axum::{ @@ -685,18 +763,20 @@ mod tests { http::{Request, StatusCode}, Router, }; - use shared::foundation::Filter; + use chrono::Utc; use shared::{ - model::{ConfigPaths, ConfigProviderDto, EpgChannel, EpgConfigDto, EpgProgramme, EpgSourceDto, PlaylistRequest, XtreamCluster}, + foundation::Filter, + model::{ + ConfigPaths, ConfigProviderDto, EpgChannel, EpgConfigDto, EpgProgramme, EpgSourceDto, EpgSourceTypeDto, + IcsDummyConfigDto, IcsEpgSourceConfigDto, PlaylistRequest, ProcessingOrder, XtreamCluster, + }, utils::Internable, }; use std::sync::Arc; - use chrono::Utc; use tempfile::tempdir; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use tower::ServiceExt; - use shared::model::ProcessingOrder; /// Generate an XMLTV datetime string in the format `YYYYMMDDHHmmss +0000` /// offset by `hours_from_now` hours from the current time. @@ -705,6 +785,22 @@ mod tests { dt.format("%Y%m%d%H%M%S %z").to_string() } + fn xmltv_source_dto(url: &str, priority: i16) -> EpgSourceDto { + EpgSourceDto { url: url.to_string(), priority, ..EpgSourceDto::default() } + } + + fn ics_source_dto(url: &str, channel_id: &str, priority: i16) -> EpgSourceDto { + EpgSourceDto { + source_type: EpgSourceTypeDto::Ics, + url: url.to_string(), + priority, + channel_id: Some(channel_id.to_string()), + channel_title: Some("Formula 1".to_string()), + ics: Some(IcsEpgSourceConfigDto::default()), + ..EpgSourceDto::default() + } + } + fn test_app_config(input: Arc, source: ConfigSource) -> AppConfig { let sources = SourcesConfig { batch_files: vec![], @@ -882,7 +978,8 @@ mod tests { let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_config = test_app_config(input, source); let original = "provider://unknown/live/user/pass/1359.ts"; - let resolved = resolve_provider_url_for_request(&app_config, &PlaylistRequest::Input("input".to_string()), original); + let resolved = + resolve_provider_url_for_request(&app_config, &PlaylistRequest::Input("input".to_string()), original); assert_eq!(resolved, original); } @@ -967,10 +1064,8 @@ mod tests { watch: None, use_memory_cache: false, }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input_a.name), Arc::clone(&input_b.name)], - targets: vec![target], - }; + let source = + ConfigSource { inputs: vec![Arc::clone(&input_a.name), Arc::clone(&input_b.name)], targets: vec![target] }; let sources = SourcesConfig { batch_files: vec![], provider: vec![], @@ -1013,7 +1108,8 @@ mod tests { #[test] fn build_playlist_webplayer_url_uses_cluster_stream_type() { let live = super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Live); - let movie = super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Video); + let movie = + super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Video); let series = super::build_playlist_webplayer_url("http://player.example", "token123", 1, 42, XtreamCluster::Series); @@ -1060,18 +1156,18 @@ mod tests { ], }; - let channels = super::merge_epg_channels(vec![(10, vec![low_priority]), (0, vec![high_priority]), (0, vec![same_priority])]); + let channels = super::merge_epg_channels(vec![ + (10, vec![low_priority]), + (0, vec![high_priority]), + (0, vec![same_priority]), + ]); assert_eq!(channels.len(), 1); assert_eq!(channels[0].title.as_deref(), Some("High")); assert_eq!(channels[0].icon.as_deref(), Some("http://high/icon.png")); assert_eq!(channels[0].programmes.len(), 3); assert_eq!( - channels[0] - .programmes - .iter() - .map(|programme| (programme.start, programme.stop)) - .collect::>(), + channels[0].programmes.iter().map(|programme| (programme.start, programme.stop)).collect::>(), vec![(10, 20), (30, 40), (50, 60)], ); } @@ -1089,37 +1185,26 @@ mod tests { id: 7, name: "input".intern(), epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![EpgSourceDto { - url: "provider://demo/xmltv.php?username=user&password=pass".to_string(), - priority: 0, - logo_override: false, - }]), - t_sources: vec![EpgSourceDto { - url: "provider://demo/xmltv.php?username=user&password=pass".to_string(), - priority: 0, - logo_override: false, - }], + sources: Some(vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)]), + t_sources: vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)], smart_match: None, })), provider_configs: Some(vec![Arc::new(provider)]), ..Default::default() }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![], - }; + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_config = test_app_config(input.clone(), source); app_config.config.store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); - let raw_epg_path = get_input_raw_epg_file_path( - "http://provider.example/xmltv.php?username=user&password=pass", + let raw_epg_path = get_input_raw_xmltv_file_path( + "provider://demo/xmltv.php?username=user&password=pass", input.as_ref(), temp_dir.path().to_string_lossy().as_ref(), ) - .await - .expect("epg path"); + .await + .expect("epg path"); if let Some(parent) = raw_epg_path.parent() { tokio::fs::create_dir_all(parent).await.expect("epg dir"); } @@ -1139,35 +1224,256 @@ mod tests { "# ), ) - .await - .expect("write epg"); + .await + .expect("write epg"); let app_state = test_app_state(Arc::new(app_config)); - let channels = super::load_epg_channels_for_input(&app_state, input.as_ref()) - .await - .expect("load epg") - .expect("channels"); + let channels = + super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); assert_eq!(channels.len(), 1); assert_eq!(channels[0].id.as_ref(), "demo.channel"); assert_eq!(channels[0].programmes.len(), 1); + assert_eq!(channels[0].programmes[0].title.as_ref().map(std::convert::AsRef::as_ref), Some("Morning Show")); + } + + #[tokio::test] + async fn load_epg_channels_for_input_reads_cached_ics_source() { + let temp_dir = tempdir().expect("temp dir"); + let ics_dto = ics_source_dto("https://example.com/f1.ics", "f1.calendar", -10); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config.config.store(Arc::new(Config { + storage_dir: temp_dir.path().to_string_lossy().to_string(), + ..Default::default() + })); + let epg_source = &input.epg.as_ref().expect("epg").sources[0]; + let raw_epg_path = + get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + tokio::fs::write( + &raw_epg_path, + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Practice 1\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .await + .expect("write ics"); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = + super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].id.as_ref(), "f1.calendar"); + assert_eq!(channels[0].title.as_deref(), Some("Formula 1")); + assert_eq!(channels[0].programmes[0].title.as_deref(), Some("Practice 1")); + } + + #[tokio::test] + async fn load_epg_channels_for_input_redownloads_invalid_cached_ics_source() { + let temp_dir = tempdir().expect("temp dir"); + tokio::fs::write( + temp_dir.path().join("valid.ics"), + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Downloaded\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .await + .expect("write valid ics source"); + let ics_dto = ics_source_dto("valid.ics", "f1.calendar", -10); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config.config.store(Arc::new(Config { + storage_dir: temp_dir.path().to_string_lossy().to_string(), + ..Default::default() + })); + let epg_source = &input.epg.as_ref().expect("epg").sources[0]; + let raw_epg_path = + get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + tokio::fs::write(&raw_epg_path, "upstream returned an error page").await.expect("write invalid cache"); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = + super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].programmes[0].title.as_deref(), Some("Downloaded")); assert_eq!( - channels[0].programmes[0].title.as_ref().map(std::convert::AsRef::as_ref), - Some("Morning Show") + tokio::fs::read_to_string(&raw_epg_path).await.expect("updated cache"), + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Downloaded\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR" ); } #[tokio::test] - async fn playlist_epg_custom_route_rejects_invalid_url_scheme() { + async fn load_epg_channels_for_input_preserves_invalid_cache_when_refresh_fails() { + let temp_dir = tempdir().expect("temp dir"); + let invalid_cache = "upstream returned an error page"; + let ics_dto = ics_source_dto("missing.ics", "f1.calendar", -10); let input = Arc::new(ConfigInput { id: 7, name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), ..Default::default() }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![], - }; + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config.config.store(Arc::new(Config { + storage_dir: temp_dir.path().to_string_lossy().to_string(), + ..Default::default() + })); + let epg_source = &input.epg.as_ref().expect("epg").sources[0]; + let raw_epg_path = + get_input_raw_epg_file_path(epg_source, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("epg path"); + if let Some(parent) = raw_epg_path.parent() { + tokio::fs::create_dir_all(parent).await.expect("epg dir"); + } + tokio::fs::write(&raw_epg_path, invalid_cache).await.expect("write invalid cache"); + + let app_state = test_app_state(Arc::new(app_config)); + assert!(super::load_epg_channels_for_input(&app_state, input.as_ref()).await.is_err()); + assert_eq!(tokio::fs::read_to_string(&raw_epg_path).await.expect("preserved cache"), invalid_cache); + } + + #[tokio::test] + async fn load_epg_channels_for_input_applies_ics_dummy_policy_in_preview() { + let temp_dir = tempdir().expect("temp dir"); + tokio::fs::write(temp_dir.path().join("empty.ics"), "BEGIN:VCALENDAR\nEND:VCALENDAR") + .await + .expect("write empty ics source"); + let mut ics_dto = ics_source_dto("empty.ics", "f1.calendar", -10); + ics_dto.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + enabled: true, + title: "No F1".to_string(), + days_past: 0, + days_future: 0, + block_hours: 24, + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![ics_dto.clone()]), + t_sources: vec![ics_dto], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config.config.store(Arc::new(Config { + storage_dir: temp_dir.path().to_string_lossy().to_string(), + ..Default::default() + })); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = + super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); + + assert_eq!(channels.len(), 1); + assert_eq!(channels[0].id.as_ref(), "f1.calendar"); + assert!(!channels[0].programmes.is_empty()); + assert!(channels[0].programmes.iter().all(|programme| programme.title.as_deref() == Some("No F1"))); + } + + #[tokio::test] + async fn load_epg_channels_for_input_prefers_high_priority_ics_dummy_policy() { + let temp_dir = tempdir().expect("temp dir"); + for filename in ["low.ics", "high.ics"] { + tokio::fs::write(temp_dir.path().join(filename), "BEGIN:VCALENDAR\nEND:VCALENDAR") + .await + .expect("write empty ics source"); + } + + let mut low_priority = ics_source_dto("low.ics", "f1.calendar", 10); + low_priority.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + enabled: true, + title: "Low priority".to_string(), + days_past: 0, + days_future: 0, + block_hours: 24, + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let mut high_priority = ics_source_dto("high.ics", "f1.calendar", -10); + high_priority.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + enabled: true, + title: "High priority".to_string(), + days_past: 0, + days_future: 0, + block_hours: 24, + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let input = Arc::new(ConfigInput { + id: 7, + name: "input".intern(), + epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { + sources: Some(vec![low_priority.clone(), high_priority.clone()]), + t_sources: vec![low_priority, high_priority], + smart_match: None, + })), + ..Default::default() + }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; + let app_config = test_app_config(input.clone(), source); + app_config.config.store(Arc::new(Config { + storage_dir: temp_dir.path().to_string_lossy().to_string(), + ..Default::default() + })); + + let app_state = test_app_state(Arc::new(app_config)); + let channels = + super::load_epg_channels_for_input(&app_state, input.as_ref()).await.expect("load epg").expect("channels"); + + assert_eq!(channels.len(), 1); + assert!(!channels[0].programmes.is_empty()); + assert!(channels[0].programmes.iter().all(|programme| programme.title.as_deref() == Some("High priority"))); + } + + #[tokio::test] + async fn playlist_epg_custom_route_rejects_invalid_url_scheme() { + let input = Arc::new(ConfigInput { id: 7, name: "input".intern(), ..Default::default() }); + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_state = test_app_state(Arc::new(test_app_config(input, source))); let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); let request = Request::builder() @@ -1195,37 +1501,26 @@ mod tests { id: 7, name: "input".intern(), epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { - sources: Some(vec![EpgSourceDto { - url: "provider://demo/xmltv.php?username=user&password=pass".to_string(), - priority: 0, - logo_override: false, - }]), - t_sources: vec![EpgSourceDto { - url: "provider://demo/xmltv.php?username=user&password=pass".to_string(), - priority: 0, - logo_override: false, - }], + sources: Some(vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)]), + t_sources: vec![xmltv_source_dto("provider://demo/xmltv.php?username=user&password=pass", 0)], smart_match: None, })), provider_configs: Some(vec![Arc::new(provider)]), ..Default::default() }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![], - }; + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_config = test_app_config(input.clone(), source); app_config.config.store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); - let raw_epg_path = get_input_raw_epg_file_path( - "http://provider.example/xmltv.php?username=user&password=pass", + let raw_epg_path = get_input_raw_xmltv_file_path( + "provider://demo/xmltv.php?username=user&password=pass", input.as_ref(), temp_dir.path().to_string_lossy().as_ref(), ) - .await - .expect("epg path"); + .await + .expect("epg path"); if let Some(parent) = raw_epg_path.parent() { tokio::fs::create_dir_all(parent).await.expect("epg dir"); } @@ -1245,8 +1540,8 @@ mod tests { "# ), ) - .await - .expect("write epg"); + .await + .expect("write epg"); let app_state = test_app_state(Arc::new(app_config)); let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); @@ -1280,10 +1575,7 @@ mod tests { provider_configs: Some(vec![Arc::new(provider)]), ..Default::default() }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![], - }; + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_state = test_app_state(Arc::new(test_app_config(input, source))); let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); let request = Request::builder() @@ -1315,75 +1607,42 @@ mod tests { name: "input".intern(), epg: Some(crate::model::EpgConfig::from(&EpgConfigDto { sources: Some(vec![ - EpgSourceDto { - url: secondary_url.to_string(), - priority: 10, - logo_override: false, - }, - EpgSourceDto { - url: primary_url.to_string(), - priority: 0, - logo_override: false, - }, - EpgSourceDto { - url: "provider://demo/xmltv-same-priority.php?username=user&password=pass".to_string(), - priority: 0, - logo_override: false, - }, + xmltv_source_dto(secondary_url, 10), + xmltv_source_dto(primary_url, 0), + xmltv_source_dto("provider://demo/xmltv-same-priority.php?username=user&password=pass", 0), ]), t_sources: vec![ - EpgSourceDto { - url: secondary_url.to_string(), - priority: 10, - logo_override: false, - }, - EpgSourceDto { - url: primary_url.to_string(), - priority: 0, - logo_override: false, - }, - EpgSourceDto { - url: "provider://demo/xmltv-same-priority.php?username=user&password=pass".to_string(), - priority: 0, - logo_override: false, - }, + xmltv_source_dto(secondary_url, 10), + xmltv_source_dto(primary_url, 0), + xmltv_source_dto("provider://demo/xmltv-same-priority.php?username=user&password=pass", 0), ], smart_match: None, })), provider_configs: Some(vec![Arc::new(provider)]), ..Default::default() }); - let source = ConfigSource { - inputs: vec![Arc::clone(&input.name)], - targets: vec![], - }; + let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![] }; let app_config = test_app_config(input.clone(), source); app_config.config.store(Arc::new(Config { storage_dir: temp_dir.path().to_string_lossy().to_string(), ..Default::default() })); - let primary_path = get_input_raw_epg_file_path( - "http://provider.example/xmltv-primary.php?username=user&password=pass", + let primary_path = + get_input_raw_xmltv_file_path(primary_url, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("primary epg path"); + let secondary_path = + get_input_raw_xmltv_file_path(secondary_url, input.as_ref(), temp_dir.path().to_string_lossy().as_ref()) + .await + .expect("secondary epg path"); + let same_priority_path = get_input_raw_xmltv_file_path( + "provider://demo/xmltv-same-priority.php?username=user&password=pass", input.as_ref(), temp_dir.path().to_string_lossy().as_ref(), ) - .await - .expect("primary epg path"); - let secondary_path = get_input_raw_epg_file_path( - "http://provider.example/xmltv-secondary.php?username=user&password=pass", - input.as_ref(), - temp_dir.path().to_string_lossy().as_ref(), - ) - .await - .expect("secondary epg path"); - let same_priority_path = get_input_raw_epg_file_path( - "http://provider.example/xmltv-same-priority.php?username=user&password=pass", - input.as_ref(), - temp_dir.path().to_string_lossy().as_ref(), - ) - .await - .expect("same priority epg path"); + .await + .expect("same priority epg path"); for path in [&primary_path, &secondary_path, &same_priority_path] { if let Some(parent) = path.parent() { @@ -1411,8 +1670,8 @@ mod tests { "# ), ) - .await - .expect("write secondary epg"); + .await + .expect("write secondary epg"); tokio::fs::write( &primary_path, format!( @@ -1428,8 +1687,8 @@ mod tests { "# ), ) - .await - .expect("write primary epg"); + .await + .expect("write primary epg"); tokio::fs::write( &same_priority_path, format!( @@ -1448,8 +1707,8 @@ mod tests { "# ), ) - .await - .expect("write same priority epg"); + .await + .expect("write same priority epg"); let app_state = test_app_state(Arc::new(app_config)); let router = super::v1_api_playlist_register_protected(Router::new()).with_state(app_state); @@ -1471,5 +1730,4 @@ mod tests { assert!(body_text.contains("Second Show"), "{body_text}"); assert!(!body_text.contains("Secondary Show"), "{body_text}"); } - } diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index 470496ea9..c3095341d 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -794,10 +794,17 @@ mod tests { get_epg_path_for_target_by_type, group_stream_epg_items, rewrite_epg_channel_resource_url, + serve_epg, stream_epg_programmes_for_channel, stream_epg_reference_map, }; - use crate::model::{Config, ConfigTarget, M3uTargetOutput, TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput}; + use crate::api::model::create_test_app_state; + use crate::model::{ + Config, ConfigTarget, Epg, IcsEpgSourceConfig, M3uTargetOutput, ProxyUserCredentials, TargetOutput, + XtreamTargetFlagsSet, XtreamTargetOutput, + }; + use crate::processing::parser::ics::parse_ics_file_to_channel; + use crate::repository::epg_write_file; use crate::utils::{EpgProcessingOptions, EpgTimeShift}; use arc_swap::ArcSwapOption; use shared::{ @@ -805,7 +812,7 @@ mod tests { model::{EpgChannel, EpgProgramme, ProcessingOrder, StreamEpgItemRequest, TargetType}, utils::{concat_path, obfuscate_text, Internable}, }; - use std::fs; + use std::{collections::HashMap, fs}; use std::sync::Arc; use tempfile::tempdir; @@ -870,6 +877,72 @@ mod tests { } } + #[tokio::test] + async fn regular_xmltv_output_contains_imported_ics_channel_and_programme() { + let dir = tempdir().expect("temp dir"); + let ics_path = dir.path().join("calendar.ics"); + fs::write( + &ics_path, + concat!( + "BEGIN:VCALENDAR\r\n", + "VERSION:2.0\r\n", + "BEGIN:VEVENT\r\n", + "UID:f1-race\r\n", + "DTSTART:20300310T140000Z\r\n", + "DTEND:20300310T160000Z\r\n", + "SUMMARY:Formula 1 Grand Prix\r\n", + "DESCRIPTION:Imported calendar programme\r\n", + "END:VEVENT\r\n", + "END:VCALENDAR\r\n", + ), + ) + .expect("write ICS fixture"); + + let channel = parse_ics_file_to_channel( + &ics_path, + "f1.calendar".intern(), + None, + &IcsEpgSourceConfig::default(), + ) + .await + .expect("parse ICS fixture"); + let epg_path = dir.path().join("epg.db"); + epg_write_file( + "ics-target", + &Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(channel)], + }, + &epg_path, + &HashMap::, Arc>::new(), + ) + .expect("write EPG database"); + + let config = test_config_with_storage(dir.path().to_string_lossy().as_ref()); + let app_state = create_test_app_state(config); + let target = Arc::new(test_target_with_xtream_and_m3u()); + let response = serve_epg( + &app_state, + &epg_path, + &ProxyUserCredentials::default(), + &target, + None, + ) + .await; + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read XMLTV response body"); + let xml = String::from_utf8(body.to_vec()).expect("XMLTV response is UTF-8"); + + assert!(xml.contains(r#""#)); + assert!(xml.contains("f1.calendar")); + assert!(xml.contains(r#"channel="f1.calendar""#)); + assert!(xml.contains("Formula 1 Grand Prix")); + assert!(xml.contains("Imported calendar programme")); + } + #[test] fn get_epg_path_for_target_by_type_prefers_requested_output() { let dir = tempdir().expect("temp dir"); diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index 050346e97..f9d44dd6a 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1700,16 +1700,35 @@ pub fn xtream_api_register() -> axum::Router> { mod tests { use super::{ empty_stream_info_response, get_xtream_player_api_stream_url, resolve_xtream_playback_extension, - ApiStreamContext, XtreamApiTimeShiftRequest, + xtream_get_short_epg, ApiStreamContext, XtreamApiTimeShiftRequest, }; - use crate::{api::model::UserApiRequest, model::ConfigInput}; + use crate::{ + api::model::{ + create_test_app_state, PlaylistStorage, PlaylistXtreamStorage, UserApiRequest, + }, + model::{ + Config, ConfigInput, ConfigTarget, Epg, IcsEpgSourceConfig, ProxyUserCredentials, TargetOutput, + XtreamTargetFlagsSet, XtreamTargetOutput, + }, + processing::parser::ics::parse_ics_file_to_channel, + repository::{ + epg_write_file, xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTree, + VirtualIdRecord, + }, + }; + use arc_swap::ArcSwapOption; + use axum::response::IntoResponse; + use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine}; + use shared::foundation::Filter; use shared::{ model::{ - InputType, PlaylistItemType, StreamProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, + ClusterFlags, InputType, PlaylistItemType, ProcessingOrder, StreamProperties, UUIDType, + VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, }, utils::Internable, }; use std::sync::Arc; + use tempfile::tempdir; async fn response_body_text(response: axum::response::Response) -> Result { let body = axum::body::to_bytes(response.into_body(), usize::MAX) @@ -1718,6 +1737,154 @@ mod tests { String::from_utf8(body.to_vec()).map_err(|err| format!("response body is not UTF-8: {err}")) } + fn short_epg_target() -> ConfigTarget { + ConfigTarget { + id: 1, + enabled: true, + name: "ics-xtream".to_string(), + options: None, + sort: None, + filter: Filter::default(), + output: vec![TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + })], + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::new(None)), + favourites: None, + processing_order: ProcessingOrder::default(), + watch: None, + use_memory_cache: true, + } + } + + fn short_epg_live_item() -> XtreamPlaylistItem { + XtreamPlaylistItem { + virtual_id: 100, + provider_id: 0, + name: "Formula 1".intern(), + logo: "".intern(), + logo_small: "".intern(), + group: "Sports".intern(), + title: "".intern(), + parent_code: "".intern(), + rec: "".intern(), + url: "http://example.invalid/live.ts".intern(), + epg_channel_id: Some("f1.calendar".intern()), + xtream_cluster: XtreamCluster::Live, + additional_properties: None, + item_type: PlaylistItemType::Live, + category_id: 1, + input_name: "local".intern(), + channel_no: 1, + source_ordinal: 0, + } + } + + #[tokio::test] + async fn xtream_short_epg_returns_imported_ics_programme_for_matching_channel_id() { + let dir = tempdir().expect("temp dir"); + let ics_path = dir.path().join("calendar.ics"); + let start = chrono::Utc::now() + chrono::Duration::days(1); + let stop = start + chrono::Duration::hours(1); + std::fs::write( + &ics_path, + format!( + concat!( + "BEGIN:VCALENDAR\r\n", + "VERSION:2.0\r\n", + "BEGIN:VEVENT\r\n", + "UID:f1-qualifying\r\n", + "DTSTART:{}\r\n", + "DTEND:{}\r\n", + "SUMMARY:Formula 1 Qualifying\r\n", + "DESCRIPTION:Imported from ICS\r\n", + "END:VEVENT\r\n", + "END:VCALENDAR\r\n", + ), + start.format("%Y%m%dT%H%M%SZ"), + stop.format("%Y%m%dT%H%M%SZ"), + ), + ) + .expect("write ICS fixture"); + let channel = parse_ics_file_to_channel( + &ics_path, + "f1.calendar".intern(), + Some("Formula 1".intern()), + &IcsEpgSourceConfig::default(), + ) + .await + .expect("parse ICS fixture"); + + let config = Config { + storage_dir: dir.path().to_string_lossy().into_owned(), + ..Config::default() + }; + let target = Arc::new(short_epg_target()); + let xtream_storage = xtream_get_storage_path(&config, &target.name).expect("xtream storage path"); + std::fs::create_dir_all(&xtream_storage).expect("create xtream storage"); + let epg_path = xtream_get_epg_file_path_for_target(&xtream_storage); + epg_write_file( + &target.name, + &Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(channel)], + }, + &epg_path, + &std::collections::HashMap::, Arc>::new(), + ) + .expect("write target EPG"); + + let app_state = create_test_app_state(config); + let live_item = short_epg_live_item(); + let mut live = BPlusTree::new(); + live.insert(live_item.virtual_id, live_item.clone()); + app_state + .playlists + .cache_playlist( + &target.name, + PlaylistStorage::XtreamPlaylist(Box::new(PlaylistXtreamStorage { + live, + vod: BPlusTree::new(), + series: BPlusTree::new(), + })), + ) + .await; + let mut id_mapping = BPlusTree::new(); + id_mapping.insert( + live_item.virtual_id, + VirtualIdRecord::new( + live_item.provider_id, + live_item.virtual_id, + PlaylistItemType::Live, + 0, + UUIDType::default(), + ), + ); + app_state.playlists.cache_id_mapping(&target.name, id_mapping).await; + + let mut user = ProxyUserCredentials::default(); + user.output_clusters = ClusterFlags::all(); + let response = xtream_get_short_epg(&app_state, &user, &target, "100", 4) + .await + .into_response(); + let body = response_body_text(response).await.expect("read short EPG response"); + let json: serde_json::Value = serde_json::from_str(&body).expect("parse short EPG JSON"); + let decode_listing_text = |field: &str| { + let encoded = json["epg_listings"][0][field].as_str().expect("encoded listing text"); + String::from_utf8(BASE64_STANDARD.decode(encoded).expect("decode listing text")) + .expect("decoded listing text is UTF-8") + }; + + assert_eq!(json["epg_listings"][0]["channel_id"], "f1.calendar"); + assert_eq!(decode_listing_text("title"), "Formula 1 Qualifying"); + assert_eq!(decode_listing_text("description"), "Imported from ICS"); + } + #[tokio::test] async fn empty_stream_info_response_uses_vod_object_and_list_shapes() -> Result<(), String> { assert_eq!(response_body_text(empty_stream_info_response(XtreamCluster::Video)).await?, "{}"); diff --git a/backend/src/api/model/app_state.rs b/backend/src/api/model/app_state.rs index e05315e45..4525e8de0 100644 --- a/backend/src/api/model/app_state.rs +++ b/backend/src/api/model/app_state.rs @@ -425,6 +425,79 @@ pub struct AppState { pub manual_update_sender: mpsc::Sender, } +#[cfg(test)] +pub(crate) fn create_test_app_state(config: Config) -> Arc { + let app_config = Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(config)), + sources: Arc::new(ArcSwap::from_pointee(SourcesConfig::default())), + hdhomerun: Arc::new(ArcSwapOption::default()), + api_proxy: Arc::new(ArcSwapOption::default()), + file_locks: Arc::new(crate::utils::FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(shared::model::ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(crate::model::MediaToolCapabilities::new()), + }); + let event_manager = Arc::new(EventManager::new()); + let active_provider = Arc::new(ActiveProviderManager::new(&app_config, &event_manager)); + let shared_stream_manager = Arc::new(SharedStreamManager::new(Arc::clone(&active_provider))); + active_provider.set_shared_stream_manager(Arc::clone(&shared_stream_manager)); + + let geoip = Arc::new(ArcSwapOption::::default()); + let loaded_config = app_config.config.load(); + let active_users = Arc::new(ActiveUserManager::new(&loaded_config, &geoip, &event_manager)); + let connection_manager = Arc::new(ConnectionManager::new( + &active_users, + &active_provider, + &shared_stream_manager, + &event_manager, + None, + )); + let tokens = CancelTokens::default(); + let metadata_manager = Arc::new(MetadataUpdateManager::new(tokens.metadata.clone())); + let (manual_update_sender, _) = mpsc::channel::(1); + + Arc::new(AppState { + forced_targets: Arc::new(ArcSwap::from_pointee(ProcessTargets { + enabled: false, + inputs: Vec::new(), + targets: Vec::new(), + target_names: Vec::new(), + })), + app_config, + http_client: Arc::new(ArcSwap::from_pointee(Client::new())), + http_client_no_redirect: Arc::new(ArcSwap::from_pointee(Client::new())), + downloads: Arc::new(DownloadQueue::new()), + cache: Arc::new(ArcSwapOption::default()), + shared_stream_manager, + hls_proxy: Arc::new(HlsProxyManager::new()), + hls_provisioning: Arc::new(HlsProvisioningState::new()), + active_users, + active_provider, + connection_manager, + event_manager, + cancel_tokens: Arc::new(ArcSwap::from_pointee(tokens)), + playlists: Arc::new(PlaylistStorageState::new()), + geoip, + update_guard: UpdateGuard::new(), + metadata_manager, + manual_update_sender, + }) +} + impl AppState { pub(in crate::api::model) async fn set_config(&self, config: Config) -> Result { let old_storage_dir = self.app_config.config.load().storage_dir.clone(); diff --git a/backend/src/api/sys_usage.rs b/backend/src/api/sys_usage.rs index a54ac8c46..e1bed7cd3 100644 --- a/backend/src/api/sys_usage.rs +++ b/backend/src/api/sys_usage.rs @@ -100,6 +100,7 @@ impl DiskProbe { /// Return `(total_bytes, free_bytes_available_to_caller)`. /// Returns `(0, 0)` if the underlying syscall fails. + #[allow(clippy::useless_conversion)] fn sample(&self) -> (u64, u64) { #[cfg(unix)] { @@ -111,8 +112,8 @@ impl DiskProbe { return (0, 0); } let bsize = stat.f_frsize as u64; - let total = (stat.f_blocks as u64).saturating_mul(bsize); - let free = (stat.f_bavail as u64).saturating_mul(bsize); + let total = u64::from(stat.f_blocks).saturating_mul(bsize); + let free = u64::from(stat.f_bavail).saturating_mul(bsize); (total, free) } #[cfg(windows)] @@ -574,14 +575,14 @@ mod platform { let memory_usage = query_process_memory_usage(self.process)?; self.networks.refresh(true); let (rx_bytes, tx_bytes) = super::sum_sysinfo_network_bytes(&self.networks); - let (net_rx_bytes_per_sec, net_tx_bytes_per_sec) = self.net_tracker.sample(rx_bytes, tx_bytes); + let (received_bps, sent_bps) = self.net_tracker.sample(rx_bytes, tx_bytes); let (disk_total_bytes, disk_free_bytes) = self.disk_probe.as_ref().map_or((0, 0), DiskProbe::sample); Some(SystemInfo { cpu_usage: self.cpu_tracker.sample(cpu_time_secs), memory_usage, memory_total: self.memory_total, - net_rx_bytes_per_sec, - net_tx_bytes_per_sec, + net_rx_bytes_per_sec: received_bps, + net_tx_bytes_per_sec: sent_bps, disk_total_bytes, disk_free_bytes, }) @@ -698,14 +699,14 @@ mod platform { let memory_usage = query_process_memory_usage()?; self.networks.refresh(true); let (rx_bytes, tx_bytes) = super::sum_sysinfo_network_bytes(&self.networks); - let (net_rx_bytes_per_sec, net_tx_bytes_per_sec) = self.net_tracker.sample(rx_bytes, tx_bytes); + let (received_bps, sent_bps) = self.net_tracker.sample(rx_bytes, tx_bytes); let (disk_total_bytes, disk_free_bytes) = self.disk_probe.as_ref().map_or((0, 0), DiskProbe::sample); Some(SystemInfo { cpu_usage: self.cpu_tracker.sample(cpu_time_secs), memory_usage, memory_total: self.memory_total, - net_rx_bytes_per_sec, - net_tx_bytes_per_sec, + net_rx_bytes_per_sec: received_bps, + net_tx_bytes_per_sec: sent_bps, disk_total_bytes, disk_free_bytes, }) diff --git a/backend/src/model/config/epg.rs b/backend/src/model/config/epg.rs index bf31cdf11..66b1dcdb9 100644 --- a/backend/src/model/config/epg.rs +++ b/backend/src/model/config/epg.rs @@ -1,24 +1,133 @@ -use shared::model::{EpgConfigDto, EpgSourceDto}; use crate::model::{macros, EpgSmartMatchConfig}; +use shared::model::{ + EpgConfigDto, EpgSourceDto, EpgSourceTypeDto, IcsDummyConfigDto, IcsEpgSourceConfigDto, + IcsEventMappingDto, +}; +use shared::utils::Internable; +use std::sync::Arc; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum EpgSourceType { + Xmltv, + Ics, +} + +impl From for EpgSourceType { + fn from(value: EpgSourceTypeDto) -> Self { + match value { + EpgSourceTypeDto::Xmltv => Self::Xmltv, + EpgSourceTypeDto::Ics => Self::Ics, + } + } +} #[derive(Debug, Clone)] pub struct EpgSource { + pub source_type: EpgSourceType, pub url: String, pub priority: i16, pub logo_override: bool, + pub channel_id: Option>, + pub channel_title: Option>, + pub match_names: Vec>, + pub ics: Option, } macros::from_impl!(EpgSource); impl From<&EpgSourceDto> for EpgSource { fn from(dto: &EpgSourceDto) -> Self { Self { + source_type: EpgSourceType::from(dto.source_type), url: dto.url.clone(), priority: dto.priority, logo_override: dto.logo_override, + channel_id: dto.channel_id.as_deref().map(Internable::intern), + channel_title: dto.channel_title.as_deref().map(Internable::intern), + match_names: dto.match_names.iter().map(|name| name.as_str().intern()).collect(), + ics: dto.ics.as_ref().map(IcsEpgSourceConfig::from), } } } +#[derive(Debug, Clone)] +pub struct IcsEpgSourceConfig { + pub timezone: String, + pub event: IcsEventMapping, + pub dummy: IcsDummyConfig, + pub include_cancelled: bool, + pub max_events: usize, + pub max_download_bytes: u64, + pub max_decompressed_bytes: usize, +} + +impl Default for IcsEpgSourceConfig { + fn default() -> Self { Self::from(&IcsEpgSourceConfigDto::default()) } +} + +impl From<&IcsEpgSourceConfigDto> for IcsEpgSourceConfig { + fn from(dto: &IcsEpgSourceConfigDto) -> Self { + Self { + timezone: dto.timezone.clone(), + event: IcsEventMapping::from(&dto.event), + dummy: IcsDummyConfig::from(&dto.dummy), + include_cancelled: dto.include_cancelled, + max_events: dto.max_events, + max_download_bytes: dto.max_download_bytes, + max_decompressed_bytes: dto.max_decompressed_bytes, + } + } +} + +#[derive(Debug, Clone)] +pub struct IcsEventMapping { + pub title: String, + pub description: String, + pub include_location: bool, + pub include_categories: bool, +} + +impl From<&IcsEventMappingDto> for IcsEventMapping { + fn from(dto: &IcsEventMappingDto) -> Self { + Self { + title: dto.title.clone(), + description: dto.description.clone(), + include_location: dto.include_location, + include_categories: dto.include_categories, + } + } +} + +#[derive(Debug, Clone)] +pub struct IcsDummyConfig { + pub enabled: bool, + pub title: String, + pub description: String, + pub days_past: u16, + pub days_future: u16, + pub block_hours: u8, + pub min_gap_minutes: u16, +} + +impl From<&IcsDummyConfigDto> for IcsDummyConfig { + fn from(dto: &IcsDummyConfigDto) -> Self { + Self { + enabled: dto.enabled, + title: dto.title.clone(), + description: dto.description.clone(), + days_past: dto.days_past, + days_future: dto.days_future, + block_hours: dto.block_hours, + min_gap_minutes: dto.min_gap_minutes, + } + } +} + +#[derive(Debug, Clone)] +pub struct IcsDummyPolicy { + pub timezone: String, + pub config: IcsDummyConfig, +} + #[derive(Debug, Clone)] pub struct EpgConfig { pub sources: Vec, @@ -34,3 +143,47 @@ impl From<&EpgConfigDto> for EpgConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + use shared::model::{EpgSourceTypeDto, IcsEpgSourceConfigDto}; + + #[test] + fn epg_source_dto_type_xmltv_maps_to_runtime_xmltv() { + let dto = EpgSourceDto { + source_type: EpgSourceTypeDto::Xmltv, + url: "https://example.com/xmltv.xml".to_string(), + ..EpgSourceDto::default() + }; + + let source = EpgSource::from(&dto); + + assert_eq!(source.source_type, EpgSourceType::Xmltv); + assert_eq!(source.url, "https://example.com/xmltv.xml"); + } + + #[test] + fn epg_source_dto_type_ics_maps_channel_metadata() { + let dto = EpgSourceDto { + source_type: EpgSourceTypeDto::Ics, + url: "https://example.com/f1.ics".to_string(), + channel_id: Some("f1.calendar".to_string()), + channel_title: Some("Formula 1".to_string()), + match_names: vec!["F1".to_string(), "Formel 1".to_string()], + ics: Some(IcsEpgSourceConfigDto::default()), + ..EpgSourceDto::default() + }; + + let source = EpgSource::from(&dto); + + assert_eq!(source.source_type, EpgSourceType::Ics); + assert_eq!(source.channel_id.as_deref(), Some("f1.calendar")); + assert_eq!(source.channel_title.as_deref(), Some("Formula 1")); + assert_eq!( + source.match_names.iter().map(AsRef::as_ref).collect::>(), + vec!["F1", "Formel 1"] + ); + assert_eq!(source.ics.as_ref().map(|config| config.max_events), Some(50_000)); + } +} diff --git a/backend/src/model/config/reverse_proxy.rs b/backend/src/model/config/reverse_proxy.rs index 210f78d23..8dd55f697 100644 --- a/backend/src/model/config/reverse_proxy.rs +++ b/backend/src/model/config/reverse_proxy.rs @@ -419,8 +419,8 @@ impl From<&ReverseProxyConfig> for ReverseProxyConfigDto { #[cfg(test)] mod tests { use super::{ - HlsCacheConfig, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, - ReverseProxyConfig, + default_hls_cache_path, HlsCacheConfig, HlsManifestRecoveryBurstConfig, HlsSegmentRepairConfig, + ReverseProxyConfig, }; use shared::model::{ByteSize, HlsCacheConfigDto, HlsManifestRecoveryBurstLevel, HlsSegmentRepairMode, HlsStripMode, QosAggregationConfigDto, ReverseProxyConfigDto, StreamHistoryConfigDto}; @@ -494,7 +494,7 @@ mod tests { assert_eq!( hls, HlsCacheConfig { - cache_path: "/tmp/tuliprox/cache/hls".to_string(), + cache_path: default_hls_cache_path(), strip: super::StripConfig { mode: HlsStripMode::Segments, value: 0, diff --git a/backend/src/model/xmltv.rs b/backend/src/model/xmltv.rs index cf7d4fbbf..c1fff3393 100644 --- a/backend/src/model/xmltv.rs +++ b/backend/src/model/xmltv.rs @@ -1,8 +1,9 @@ use crate::api::model::AppState; +use crate::model::IcsEpgSourceConfig; use crate::model::xmltv::XmlTagIcon::Undefined; use crate::model::InputSource; use crate::utils::request::get_remote_content_as_stream; -use crate::utils::{async_file_reader, parse_xmltv_time}; +use crate::utils::{async_file_reader, parse_xmltv_time, FileLockManager}; use chrono::{Datelike, TimeZone, Utc}; use futures::TryFutureExt; use quick_xml::events::{Event}; @@ -77,30 +78,49 @@ pub struct Epg { pub children: Vec>, } +#[derive(Debug, Clone)] +pub enum PersistedEpgSourceKind { + Xmltv, + Ics { + channel_id: Arc, + channel_title: Option>, + match_names: Vec>, + config: Box, + }, +} + #[derive(Debug, Clone)] pub struct PersistedEpgSource { pub file_path: PathBuf, pub priority: i16, pub logo_override: bool, + pub kind: PersistedEpgSourceKind, } #[derive(Debug, Clone)] pub struct TVGuide { epg_sources: Vec, + file_locks: Option>, } impl TVGuide { pub fn new(mut epg_sources: Vec) -> Self { epg_sources.sort_by_key(|a| a.priority); - Self { - epg_sources, - } + Self { epg_sources, file_locks: None } + } + + /// Uses the shared cache lock manager while reading persisted EPG sources. + pub fn with_file_locks(mut self, file_locks: Arc) -> Self { + self.file_locks = Some(file_locks); + self } #[inline] pub fn get_epg_sources(&self) -> &Vec { &self.epg_sources } + + pub(crate) fn get_file_locks(&self) -> Option<&FileLockManager> { self.file_locks.as_deref() } } diff --git a/backend/src/processing/parser/ics/dummy.rs b/backend/src/processing/parser/ics/dummy.rs new file mode 100644 index 000000000..c301aa413 --- /dev/null +++ b/backend/src/processing/parser/ics/dummy.rs @@ -0,0 +1,458 @@ +use crate::model::IcsDummyConfig; +use chrono::{Duration, LocalResult, NaiveDate, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::Tz; +use shared::{error::TuliproxError, model::EpgProgramme, utils::Internable}; +use std::sync::Arc; + +pub fn fill_dummy_gaps( + programmes: &mut Vec, + channel_id: &Arc, + timezone: &str, + config: &IcsDummyConfig, + now: chrono::DateTime, +) -> Result<(), TuliproxError> { + if !config.enabled { + return Ok(()); + } + if config.block_hours == 0 || 24 % config.block_hours != 0 { + return Err(TuliproxError::ConfigEpg(format!( + "ICS dummy block_hours must divide 24 evenly, got {}", + config.block_hours + ))); + } + + let tz: Tz = + timezone.parse().map_err(|_| TuliproxError::ConfigEpg(format!("Unknown ICS timezone '{timezone}'")))?; + + let local_today = now.with_timezone(&tz).date_naive(); + let start_day = local_today - Duration::days(i64::from(config.days_past)); + let end_day = local_today + Duration::days(i64::from(config.days_future)); + let blocks = build_block_ranges(tz, start_day, end_day, config.block_hours)?; + let Some(&(window_start, _)) = blocks.first() else { + return Ok(()); + }; + let window_end = blocks.last().map_or(window_start, |(_, stop)| *stop); + + programmes.sort_by_key(|programme| (programme.start, programme.stop)); + let merged_real_ranges = merge_ranges_in_window(programmes, window_start, window_end); + let mut dummy_programmes = Vec::new(); + let mut range_index = 0; + for (block_start, block_end) in blocks { + fill_block( + &mut dummy_programmes, + &merged_real_ranges, + &mut range_index, + channel_id, + block_start, + block_end, + config, + ); + } + + programmes.extend(dummy_programmes); + programmes.sort_by_key(|programme| (programme.start, programme.stop)); + Ok(()) +} + +fn build_block_ranges( + timezone: Tz, + start_day: NaiveDate, + end_day: NaiveDate, + block_hours: u8, +) -> Result, TuliproxError> { + let mut blocks = Vec::new(); + let mut day = start_day; + while day <= end_day { + let mut boundaries = Vec::with_capacity(usize::from(24 / block_hours) + 1); + for hour in (0_u32..=24).step_by(usize::from(block_hours)) { + let local = if hour == 24 { + (day + Duration::days(1)).and_hms_opt(0, 0, 0) + } else { + day.and_hms_opt(hour, 0, 0) + } + .ok_or_else(|| TuliproxError::ConfigEpg(format!("Invalid dummy block boundary for {day} {hour}:00")))?; + boundaries.push(resolve_local_boundary(timezone, local)?); + } + + for boundary in boundaries.windows(2) { + let (start, stop) = (boundary[0], boundary[1]); + // A spring-forward gap can map two nominal boundaries to the same + // real instant. It therefore represents no programme interval. + if stop > start { + blocks.push((start, stop)); + } + } + day += Duration::days(1); + } + Ok(blocks) +} + +fn resolve_local_boundary(timezone: Tz, local: NaiveDateTime) -> Result { + match timezone.from_local_datetime(&local) { + LocalResult::Single(value) => Ok(value.with_timezone(&Utc).timestamp()), + LocalResult::Ambiguous(first, second) => Ok(first.min(second).with_timezone(&Utc).timestamp()), + LocalResult::None => { + // Map a non-existent wall-clock boundary to the first real instant + // after the DST gap. Adjacent UTC intervals stay monotone, while the + // skipped local interval becomes a zero-length block and is omitted. + let mut candidate = local; + for _ in 0..=(24 * 60) { + candidate = candidate + .checked_add_signed(Duration::minutes(1)) + .ok_or_else(|| TuliproxError::ConfigEpg(format!("Invalid local dummy time {local}")))?; + match timezone.from_local_datetime(&candidate) { + LocalResult::Single(value) => return Ok(value.with_timezone(&Utc).timestamp()), + LocalResult::Ambiguous(first, second) => { + return Ok(first.min(second).with_timezone(&Utc).timestamp()); + } + LocalResult::None => {} + } + } + Err(TuliproxError::ConfigEpg(format!("Could not resolve local dummy time {local}"))) + } + } +} + +fn merge_ranges_in_window(programmes: &[EpgProgramme], window_start: i64, window_end: i64) -> Vec<(i64, i64)> { + let mut merged = Vec::<(i64, i64)>::new(); + for programme in programmes { + if programme.stop <= window_start || programme.stop <= programme.start { + continue; + } + if programme.start >= window_end { + break; + } + + let start = programme.start.max(window_start); + let stop = programme.stop.min(window_end); + if let Some(last) = merged.last_mut() { + if start <= last.1 { + last.1 = last.1.max(stop); + continue; + } + } + merged.push((start, stop)); + } + merged +} + +fn fill_block( + out: &mut Vec, + real_ranges: &[(i64, i64)], + range_index: &mut usize, + channel_id: &Arc, + block_start: i64, + block_end: i64, + config: &IcsDummyConfig, +) { + if block_end <= block_start { + return; + } + + let mut cursor = block_start; + let min_gap_seconds = i64::from(config.min_gap_minutes) * 60; + while *range_index < real_ranges.len() && real_ranges[*range_index].1 <= block_start { + *range_index += 1; + } + + let mut current_index = *range_index; + while let Some(&(event_start, event_stop)) = real_ranges.get(current_index) { + if event_start >= block_end { + break; + } + + let clipped_start = event_start.max(block_start); + let clipped_stop = event_stop.min(block_end); + + if clipped_start > cursor { + push_dummy_if_large_enough(out, channel_id, cursor, clipped_start, config, min_gap_seconds); + } + + cursor = cursor.max(clipped_stop); + if event_stop <= block_end { + current_index += 1; + } + if cursor >= block_end { + break; + } + } + *range_index = current_index; + + if cursor < block_end { + push_dummy_if_large_enough(out, channel_id, cursor, block_end, config, min_gap_seconds); + } +} + +fn push_dummy_if_large_enough( + out: &mut Vec, + channel_id: &Arc, + start: i64, + stop: i64, + config: &IcsDummyConfig, + min_gap_seconds: i64, +) { + if stop - start < min_gap_seconds { + return; + } + + out.push(EpgProgramme::new_all( + start, + stop, + Arc::clone(channel_id), + Some(config.title.as_str().intern()), + (!config.description.is_empty()).then(|| config.description.as_str().intern()), + None, + )); +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::{TimeZone, Utc}; + use shared::model::EpgProgramme; + + fn config() -> IcsDummyConfig { + IcsDummyConfig { + enabled: true, + title: "No programme".to_string(), + description: String::new(), + days_past: 0, + days_future: 0, + block_hours: 4, + min_gap_minutes: 1, + } + } + + fn ts(hour: u32, minute: u32) -> i64 { + Utc.with_ymd_and_hms(2026, 3, 6, hour, minute, 0).single().expect("valid time").timestamp() + } + + fn real(start_hour: u32, start_minute: u32, stop_hour: u32, stop_minute: u32) -> EpgProgramme { + EpgProgramme::new(ts(start_hour, start_minute), ts(stop_hour, stop_minute), "demo".intern()) + } + + fn dummy_ranges(programmes: &[EpgProgramme]) -> Vec<(i64, i64)> { + programmes + .iter() + .filter(|programme| programme.title.as_deref() == Some("No programme")) + .map(|programme| (programme.start, programme.stop)) + .collect() + } + + fn assert_dummy_covers_local_day( + programmes: &[EpgProgramme], + timezone: &str, + date: NaiveDate, + expected_seconds: i64, + ) { + let timezone: Tz = timezone.parse().expect("timezone"); + let start = resolve_local_boundary(timezone, date.and_hms_opt(0, 0, 0).expect("midnight")).expect("day start"); + let stop = + resolve_local_boundary(timezone, (date + Duration::days(1)).and_hms_opt(0, 0, 0).expect("next midnight")) + .expect("day stop"); + let ranges = dummy_ranges(programmes); + + assert_eq!(ranges.first().map(|range| range.0), Some(start)); + assert_eq!(ranges.last().map(|range| range.1), Some(stop)); + assert!(ranges.iter().all(|(range_start, range_stop)| range_stop > range_start)); + assert!(ranges.windows(2).all(|window| window[0].1 == window[1].0)); + assert_eq!( + ranges.iter().map(|(range_start, range_stop)| range_stop - range_start).sum::(), + expected_seconds + ); + } + + #[test] + fn creates_six_dummy_programmes_for_empty_day() { + let mut programmes = Vec::new(); + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "UTC", + &config(), + Utc.with_ymd_and_hms(2026, 3, 6, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + assert_eq!(dummy_ranges(&programmes).len(), 6); + assert_eq!(dummy_ranges(&programmes)[0], (ts(0, 0), ts(4, 0))); + } + + #[test] + fn event_inside_block_splits_dummy_ranges() { + let mut programmes = vec![real(3, 30, 5, 0)]; + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "UTC", + &config(), + Utc.with_ymd_and_hms(2026, 3, 6, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + let ranges = dummy_ranges(&programmes); + assert!(ranges.contains(&(ts(0, 0), ts(3, 30)))); + assert!(ranges.contains(&(ts(5, 0), ts(8, 0)))); + } + + #[test] + fn block_boundary_event_keeps_boundary_dummy() { + let mut programmes = vec![real(4, 0, 6, 0)]; + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "UTC", + &config(), + Utc.with_ymd_and_hms(2026, 3, 6, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + let ranges = dummy_ranges(&programmes); + assert!(ranges.contains(&(ts(0, 0), ts(4, 0)))); + assert!(ranges.contains(&(ts(6, 0), ts(8, 0)))); + } + + #[test] + fn adjacent_events_do_not_get_dummy_between_them() { + let mut programmes = vec![real(10, 0, 11, 0), real(11, 0, 12, 0)]; + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "UTC", + &config(), + Utc.with_ymd_and_hms(2026, 3, 6, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + let ranges = dummy_ranges(&programmes); + assert!(ranges.contains(&(ts(8, 0), ts(10, 0)))); + assert!(!ranges.contains(&(ts(11, 0), ts(12, 0)))); + assert!(ranges.contains(&(ts(12, 0), ts(16, 0)))); + } + + #[test] + fn min_gap_minutes_skips_tiny_gaps() { + let mut cfg = config(); + cfg.min_gap_minutes = 10; + let mut programmes = vec![EpgProgramme::new(ts(0, 5), ts(4, 0), "demo".intern())]; + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "UTC", + &cfg, + Utc.with_ymd_and_hms(2026, 3, 6, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + assert!(!dummy_ranges(&programmes).contains(&(ts(0, 0), ts(0, 5)))); + } + + #[test] + fn spring_forward_with_one_hour_blocks_covers_real_day_without_zero_length_programmes() { + let mut cfg = config(); + cfg.block_hours = 1; + let mut programmes = Vec::new(); + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "Europe/Berlin", + &cfg, + Utc.with_ymd_and_hms(2026, 3, 29, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + assert_eq!(dummy_ranges(&programmes).len(), 23); + assert_dummy_covers_local_day( + &programmes, + "Europe/Berlin", + NaiveDate::from_ymd_opt(2026, 3, 29).unwrap(), + 23 * 60 * 60, + ); + } + + #[test] + fn spring_forward_with_two_hour_blocks_covers_real_day() { + let mut cfg = config(); + cfg.block_hours = 2; + let mut programmes = Vec::new(); + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "Europe/Berlin", + &cfg, + Utc.with_ymd_and_hms(2026, 3, 29, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + assert_eq!(dummy_ranges(&programmes).len(), 12); + assert_dummy_covers_local_day( + &programmes, + "Europe/Berlin", + NaiveDate::from_ymd_opt(2026, 3, 29).unwrap(), + 23 * 60 * 60, + ); + } + + #[test] + fn fall_back_with_one_hour_blocks_covers_repeated_hour_once_without_overlap() { + let mut cfg = config(); + cfg.block_hours = 1; + let mut programmes = Vec::new(); + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "Europe/Berlin", + &cfg, + Utc.with_ymd_and_hms(2026, 10, 25, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + assert_eq!(dummy_ranges(&programmes).len(), 24); + assert_dummy_covers_local_day( + &programmes, + "Europe/Berlin", + NaiveDate::from_ymd_opt(2026, 10, 25).unwrap(), + 25 * 60 * 60, + ); + } + + #[test] + fn default_four_hour_blocks_remain_stable_on_spring_forward_day() { + let mut programmes = Vec::new(); + fill_dummy_gaps( + &mut programmes, + &"demo".intern(), + "Europe/Berlin", + &config(), + Utc.with_ymd_and_hms(2026, 3, 29, 12, 0, 0).single().unwrap(), + ) + .expect("dummy"); + + assert_eq!(dummy_ranges(&programmes).len(), 6); + assert_dummy_covers_local_day( + &programmes, + "Europe/Berlin", + NaiveDate::from_ymd_opt(2026, 3, 29).unwrap(), + 23 * 60 * 60, + ); + } + + #[test] + fn merge_ranges_discards_many_events_outside_dummy_window() { + let window_start = ts(0, 0); + let window_end = Utc.with_ymd_and_hms(2026, 3, 7, 0, 0, 0).single().unwrap().timestamp(); + let mut programmes = Vec::with_capacity(20_001); + for index in 1..=10_000_i64 { + let stop = window_start - index * 120; + programmes.push(EpgProgramme::new(stop - 60, stop, "demo".intern())); + } + programmes.push(EpgProgramme::new(ts(10, 0), ts(11, 0), "demo".intern())); + for index in 1..=10_000_i64 { + let start = window_end + index * 120; + programmes.push(EpgProgramme::new(start, start + 60, "demo".intern())); + } + programmes.sort_by_key(|programme| (programme.start, programme.stop)); + + assert_eq!(merge_ranges_in_window(&programmes, window_start, window_end), vec![(ts(10, 0), ts(11, 0))]); + } +} diff --git a/backend/src/processing/parser/ics/event.rs b/backend/src/processing/parser/ics/event.rs new file mode 100644 index 000000000..56210b7fa --- /dev/null +++ b/backend/src/processing/parser/ics/event.rs @@ -0,0 +1,603 @@ +use crate::{ + model::IcsEpgSourceConfig, + processing::parser::ics::time::{display_from_timestamp_in_timezone, parse_ics_datetime, parse_ics_duration}, +}; +use chrono_tz::Tz; +use log::warn; +use shared::{ + defaults::{MAX_ICS_DESCRIPTION_LENGTH, MAX_ICS_LINE_LENGTH, MAX_ICS_PROPERTIES_PER_EVENT, MAX_ICS_SUMMARY_LENGTH}, + error::TuliproxError, +}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Default)] +pub struct IcsEvent { + pub uid: Option, + pub summary: Option, + pub description: Option, + pub location: Option, + pub categories: Option, + pub start: Option, + pub stop: Option, + pub start_display: Option, + pub stop_display: Option, + pub cancelled: bool, + pub unsupported_recurrence: bool, +} + +#[derive(Debug, Clone)] +pub struct IcsProperty { + pub name: String, + pub params: IcsPropertyParams, + pub value: String, +} + +pub type IcsPropertyParams = HashMap; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum CalendarState { + Before, + Inside, + After, +} + +pub fn parse_ics_events(content: &str, config: &IcsEpgSourceConfig) -> Result, TuliproxError> { + let lines = unfold_ics_lines(content)?; + let mut events = Vec::new(); + let mut current = Vec::::new(); + let mut calendar_state = CalendarState::Before; + let mut event_depth = 0usize; + let mut collect_current = false; + let mut current_malformed = false; + let mut examined_events = 0usize; + let mut invalid_events = 0usize; + let mut unsupported_recurrence_events = 0usize; + let mut event_limit_reached = false; + + for line in lines { + if is_component_boundary(&line, "BEGIN", "VCALENDAR") { + if calendar_state != CalendarState::Before { + return Err(invalid_calendar_structure("nested or multiple BEGIN:VCALENDAR")); + } + calendar_state = CalendarState::Inside; + continue; + } + + if is_component_boundary(&line, "END", "VCALENDAR") { + if calendar_state != CalendarState::Inside { + return Err(invalid_calendar_structure("END:VCALENDAR without matching BEGIN:VCALENDAR")); + } + if event_depth != 0 { + return Err(invalid_calendar_structure("END:VCALENDAR before END:VEVENT")); + } + calendar_state = CalendarState::After; + continue; + } + + if line.trim().is_empty() && event_depth == 0 { + continue; + } + + match calendar_state { + CalendarState::Before => { + return Err(invalid_calendar_structure("content before BEGIN:VCALENDAR")); + } + CalendarState::After => { + return Err(invalid_calendar_structure("content after END:VCALENDAR")); + } + CalendarState::Inside => {} + } + + if is_component_boundary(&line, "BEGIN", "VEVENT") { + let within_budget = examined_events < config.max_events; + if within_budget { + examined_events += 1; + } else { + event_limit_reached = true; + } + + if event_depth == 0 { + current.clear(); + current_malformed = false; + collect_current = within_budget; + } else { + current_malformed |= collect_current; + if !within_budget { + collect_current = false; + current.clear(); + } + } + event_depth += 1; + continue; + } + + if is_component_boundary(&line, "END", "VEVENT") { + if event_depth == 0 { + invalid_events += 1; + continue; + } + + event_depth -= 1; + if event_depth == 0 { + if collect_current { + if current_malformed { + invalid_events += 1; + } else { + collect_parsed_event( + ¤t, + config, + &mut events, + &mut invalid_events, + &mut unsupported_recurrence_events, + ); + } + } + collect_current = false; + current_malformed = false; + current.clear(); + } + continue; + } + + if event_depth != 0 && collect_current && !current_malformed { + current.push(line); + } + } + + validate_complete_calendar(calendar_state)?; + log_parse_summary(invalid_events, unsupported_recurrence_events, examined_events, event_limit_reached, config.max_events); + + Ok(events) +} + +fn collect_parsed_event( + lines: &[String], + config: &IcsEpgSourceConfig, + events: &mut Vec, + invalid_events: &mut usize, + unsupported_recurrence_events: &mut usize, +) { + match parse_event_lines(lines, config) { + Ok(Some(event)) => { + if event.unsupported_recurrence { + *unsupported_recurrence_events += 1; + } + events.push(event); + } + Ok(None) => {} + Err(_) => *invalid_events += 1, + } +} + +fn validate_complete_calendar(state: CalendarState) -> Result<(), TuliproxError> { + match state { + CalendarState::Before => Err(invalid_calendar_structure("missing BEGIN:VCALENDAR")), + CalendarState::Inside => Err(invalid_calendar_structure("missing END:VCALENDAR")), + CalendarState::After => Ok(()), + } +} + +fn log_parse_summary( + invalid_events: usize, + unsupported_recurrence_events: usize, + examined_events: usize, + event_limit_reached: bool, + max_events: usize, +) { + if invalid_events != 0 { + warn!("Skipped {invalid_events} invalid ICS VEVENT block(s) while examining {examined_events} block(s)"); + } + if event_limit_reached { + warn!("ICS VEVENT processing limit {max_events} reached; additional event blocks were not examined"); + } + if unsupported_recurrence_events != 0 { + warn!( + "Detected {unsupported_recurrence_events} ICS VEVENT block(s) with unsupported recurrence properties \ + (RRULE/RDATE/EXDATE); importing only their base DTSTART/DTEND occurrence" + ); + } +} + +fn invalid_calendar_structure(reason: &str) -> TuliproxError { + TuliproxError::Parse(format!("Invalid ICS VCALENDAR structure: {reason}")) +} + +fn is_component_boundary(line: &str, boundary: &str, component: &str) -> bool { + let Some((left, value)) = line.split_once(':') else { + return false; + }; + let name = left.split(';').next().unwrap_or_default().trim(); + name.eq_ignore_ascii_case(boundary) && value.eq_ignore_ascii_case(component) +} + +fn unfold_ics_lines(input: &str) -> Result, TuliproxError> { + let input = input.strip_prefix('\u{feff}').unwrap_or(input); + let normalized = input.replace("\r\n", "\n").replace('\r', "\n"); + let mut result: Vec = Vec::new(); + + for line in normalized.lines() { + if line.len() > MAX_ICS_LINE_LENGTH { + return Err(TuliproxError::Parse(format!( + "ICS line exceeds max line length of {MAX_ICS_LINE_LENGTH} bytes" + ))); + } + + if line.starts_with(' ') || line.starts_with('\t') { + let Some(last) = result.last_mut() else { + return Err(TuliproxError::Parse("ICS folded line has no preceding content line".to_string())); + }; + last.push_str(&line[1..]); + if last.len() > MAX_ICS_LINE_LENGTH { + return Err(TuliproxError::Parse(format!( + "ICS unfolded line exceeds max line length of {MAX_ICS_LINE_LENGTH} bytes" + ))); + } + } else { + result.push(line.to_string()); + } + } + + Ok(result) +} + +fn parse_event_lines(lines: &[String], config: &IcsEpgSourceConfig) -> Result, TuliproxError> { + let mut event = IcsEvent::default(); + let mut duration_seconds: Option = None; + let mut all_day = false; + let mut property_count = 0usize; + let mut start_display_timezone: Option = None; + + for line in lines { + let Some(property) = parse_property(line) else { + continue; + }; + + property_count += 1; + if property_count > MAX_ICS_PROPERTIES_PER_EVENT { + return Err(TuliproxError::Parse(format!( + "VEVENT has more than {MAX_ICS_PROPERTIES_PER_EVENT} properties" + ))); + } + + match property.name.as_str() { + "UID" => event.uid = Some(unescape_ics_text(&property.value)), + "SUMMARY" => { + event.summary = + Some(truncate_to_byte_limit(unescape_ics_text(&property.value), MAX_ICS_SUMMARY_LENGTH)); + } + "DESCRIPTION" => { + event.description = + Some(truncate_to_byte_limit(unescape_ics_text(&property.value), MAX_ICS_DESCRIPTION_LENGTH)); + } + "LOCATION" => event.location = Some(unescape_ics_text(&property.value)), + "CATEGORIES" => event.categories = Some(unescape_ics_text(&property.value)), + "STATUS" if property.value.eq_ignore_ascii_case("CANCELLED") => event.cancelled = true, + "RRULE" | "RDATE" | "EXDATE" => event.unsupported_recurrence = true, + "DTSTART" => { + let parsed = parse_ics_datetime(&property.params, &property.value, &config.timezone)?; + all_day |= parsed.all_day; + start_display_timezone = Some(parsed.display_timezone); + event.start = Some(parsed.timestamp); + event.start_display = Some(parsed.display); + } + "DTEND" => { + let parsed = parse_ics_datetime(&property.params, &property.value, &config.timezone)?; + all_day |= parsed.all_day; + event.stop = Some(parsed.timestamp); + event.stop_display = Some(parsed.display); + } + "DURATION" => duration_seconds = parse_ics_duration(&property.value), + _ => {} + } + } + + if all_day { + return Ok(None); + } + + if event.stop.is_none() { + if let (Some(start), Some(duration)) = (event.start, duration_seconds) { + let stop = start.checked_add(duration).ok_or_else(|| { + TuliproxError::Parse("VEVENT start plus DURATION exceeds timestamp range".to_string()) + })?; + let stop_display = start_display_timezone + .and_then(|timezone| display_from_timestamp_in_timezone(stop, timezone)) + .ok_or_else(|| { + TuliproxError::Parse( + "VEVENT start plus DURATION is outside the supported datetime range".to_string(), + ) + })?; + event.stop = Some(stop); + // DURATION has no own timezone. Keep {end} in the same display timezone as DTSTART. + event.stop_display = Some(stop_display); + } + } + + if event.start.is_none() || event.stop.is_none() { + return Ok(None); + } + + Ok(Some(event)) +} + +fn truncate_to_byte_limit(mut value: String, max_len: usize) -> String { + if value.len() <= max_len { + return value; + } + + // Parser limits are byte-oriented because input size controls cost; truncate on a UTF-8 boundary. + let mut end = max_len; + while !value.is_char_boundary(end) { + end -= 1; + } + value.truncate(end); + value +} + +fn parse_property(line: &str) -> Option { + let (left, value) = line.split_once(':')?; + let mut parts = left.split(';'); + let name = parts.next()?.trim().to_ascii_uppercase(); + let mut params = IcsPropertyParams::new(); + + for part in parts { + if let Some((key, value)) = part.split_once('=') { + params.insert(key.trim().to_ascii_uppercase(), value.trim_matches('"').to_string()); + } + } + + Some(IcsProperty { name, params, value: value.to_string() }) +} + +fn unescape_ics_text(value: &str) -> String { + let mut result = String::with_capacity(value.len()); + let mut chars = value.chars(); + + while let Some(character) = chars.next() { + if character != '\\' { + result.push(character); + continue; + } + + match chars.next() { + Some('n' | 'N') => result.push('\n'), + Some(',') => result.push(','), + Some(';') => result.push(';'), + Some('\\') | None => result.push('\\'), + Some(next) => { + result.push('\\'); + result.push(next); + } + } + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::IcsEpgSourceConfig; + use std::fmt::Write; + + fn config() -> IcsEpgSourceConfig { IcsEpgSourceConfig::default() } + + fn calendar(body: &str) -> String { format!("BEGIN:VCALENDAR\n{body}\nEND:VCALENDAR") } + + #[test] + fn parses_utc_event_with_start_and_end() { + let content = calendar( + "BEGIN:VEVENT\nUID:1\nSUMMARY:Practice 1\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("Practice 1")); + assert_eq!(events[0].start, Some(1_772_800_200)); + assert_eq!(events[0].stop, Some(1_772_803_800)); + } + + #[test] + fn duration_supplies_missing_end() { + let content = + calendar("BEGIN:VEVENT\nSUMMARY:Qualifying\nDTSTART:20260306T123000Z\nDURATION:PT90M\nEND:VEVENT"); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].stop, Some(1_772_805_600)); + } + + #[test] + fn duration_uses_start_timezone_for_end_display() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:Qualifying\nDTSTART;TZID=Europe/Berlin:20260306T123000\nDURATION:PT90M\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].start_display.as_deref(), Some("2026-03-06T12:30:00+01:00")); + assert_eq!(events[0].stop_display.as_deref(), Some("2026-03-06T14:00:00+01:00")); + } + + #[test] + fn folded_lines_are_unfolded_and_text_is_unescaped() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:Long\n DESCRIPTION\nDESCRIPTION:Line 1\\nLine 2\\, ok\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events[0].summary.as_deref(), Some("LongDESCRIPTION")); + assert_eq!(events[0].description.as_deref(), Some("Line 1\nLine 2, ok")); + } + + #[test] + fn text_unescaping_decodes_only_known_sequences() { + assert_eq!(unescape_ics_text(r"line\nnext\Ncomma\,semi\;slash\\"), "line\nnext\ncomma,semi;slash\\"); + assert_eq!(unescape_ics_text(r"literal\\n literal\\, literal\\;"), r"literal\n literal\, literal\;"); + assert_eq!(unescape_ics_text(r"unknown\x trailing\"), r"unknown\x trailing\"); + } + + #[test] + fn event_without_start_or_end_is_skipped() { + let missing_start = calendar("BEGIN:VEVENT\nSUMMARY:No start\nDTEND:20260306T133000Z\nEND:VEVENT"); + let missing_end = calendar("BEGIN:VEVENT\nSUMMARY:No end\nDTSTART:20260306T123000Z\nEND:VEVENT"); + assert!(parse_ics_events(&missing_start, &config()).expect("events").is_empty()); + assert!(parse_ics_events(&missing_end, &config()).expect("events").is_empty()); + } + + #[test] + fn malformed_event_is_skipped_while_following_event_is_imported() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:Bad\nDTSTART:bad\nDTEND:20260306T133000Z\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:Good\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("Good")); + } + + #[test] + fn mixed_case_vevent_boundaries_are_recognized() { + let content = calendar( + "begin:VEVENT\nSUMMARY:Lower\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nend:VEVENT\nBegin:Vevent\nSUMMARY:Mixed\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!( + events.iter().filter_map(|event| event.summary.as_deref()).collect::>(), + vec!["Lower", "Mixed"] + ); + } + + #[test] + fn unknown_event_tzid_skips_only_that_event() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:Bad TZ\nDTSTART;TZID=Mars/Olympus:20260306T123000\nDTEND;TZID=Mars/Olympus:20260306T133000\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:Good\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("Good")); + } + + #[test] + fn max_events_is_enforced_without_rrule_expansion() { + let cfg = IcsEpgSourceConfig { max_events: 1, ..IcsEpgSourceConfig::default() }; + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:One\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nRRULE:FREQ=DAILY;COUNT=10\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:Two\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &cfg).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("One")); + } + + #[test] + fn recurring_event_properties_are_detected_without_expansion() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:Recurring\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nRRULE:FREQ=DAILY;COUNT=10\nEXDATE:20260307T123000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert!(events[0].unsupported_recurrence); + } + + #[test] + fn invalid_events_consume_the_same_event_budget_as_valid_events() { + let cfg = IcsEpgSourceConfig { max_events: 2, ..IcsEpgSourceConfig::default() }; + let only_invalid_before_valid = calendar( + "BEGIN:VEVENT\nDTSTART:bad-1\nDTEND:bad-1\nEND:VEVENT\nBEGIN:VEVENT\nDTSTART:bad-2\nDTEND:bad-2\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:After limit\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + assert!(parse_ics_events(&only_invalid_before_valid, &cfg).expect("events").is_empty()); + + let mixed = calendar( + "BEGIN:VEVENT\nDTSTART:bad\nDTEND:bad\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:Within budget\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:After limit\nDTSTART:20260308T123000Z\nDTEND:20260308T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&mixed, &cfg).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("Within budget")); + } + + #[test] + fn nested_vevent_boundaries_consume_the_event_budget() { + let cfg = IcsEpgSourceConfig { max_events: 2, ..IcsEpgSourceConfig::default() }; + let content = calendar( + "BEGIN:VEVENT\nBEGIN:VEVENT\nDTSTART:bad\nEND:VEVENT\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:After nested limit\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + assert!(parse_ics_events(&content, &cfg).expect("events").is_empty()); + } + + #[test] + fn all_day_events_are_skipped() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:All day\nDTSTART;VALUE=DATE:20260306\nDTEND;VALUE=DATE:20260307\nEND:VEVENT", + ); + assert!(parse_ics_events(&content, &config()).expect("events").is_empty()); + } + + #[test] + fn event_with_too_many_properties_is_skipped() { + let mut body = String::from("BEGIN:VEVENT\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\n"); + for idx in 0..=MAX_ICS_PROPERTIES_PER_EVENT { + writeln!(body, "X-PROP-{idx}:value").expect("write property fixture"); + } + body.push_str( + "END:VEVENT\nBEGIN:VEVENT\nSUMMARY:Good\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + let content = calendar(&body); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("Good")); + } + + #[test] + fn too_long_lines_fail_source_parse() { + let content = calendar(&format!("BEGIN:VEVENT\nSUMMARY:{}\nEND:VEVENT", "x".repeat(MAX_ICS_LINE_LENGTH + 1))); + let err = parse_ics_events(&content, &config()).unwrap_err(); + assert!(err.to_string().contains("max line length")); + } + + #[test] + fn overlong_summary_and_description_are_truncated() { + let content = calendar(&format!( + "BEGIN:VEVENT\nSUMMARY:{}\nDESCRIPTION:{}\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT", + "s".repeat(MAX_ICS_SUMMARY_LENGTH + 10), + "d".repeat(MAX_ICS_DESCRIPTION_LENGTH + 10), + )); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref().map(str::len), Some(MAX_ICS_SUMMARY_LENGTH)); + assert_eq!(events[0].description.as_deref().map(str::len), Some(MAX_ICS_DESCRIPTION_LENGTH)); + } + + #[test] + fn valid_empty_and_case_insensitive_bom_calendars_are_allowed() { + assert!(parse_ics_events("BEGIN:VCALENDAR\nEND:VCALENDAR", &config()).expect("empty calendar").is_empty()); + assert!(parse_ics_events("\u{feff}begin:vcalendar\nend:vcalendar", &config()) + .expect("case-insensitive calendar with BOM") + .is_empty()); + } + + #[test] + fn non_calendar_text_and_html_are_rejected() { + for content in ["garbage", "upstream error"] { + let err = parse_ics_events(content, &config()).unwrap_err(); + assert!(err.to_string().contains("VCALENDAR structure")); + } + } + + #[test] + fn incomplete_reversed_and_nested_calendar_envelopes_are_rejected() { + for content in [ + "BEGIN:VCALENDAR\nVERSION:2.0", + "END:VCALENDAR\nBEGIN:VCALENDAR", + "BEGIN:VCALENDAR\nBEGIN:VCALENDAR\nEND:VCALENDAR\nEND:VCALENDAR", + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nDTSTART:20260306T123000Z\nEND:VCALENDAR", + ] { + let err = parse_ics_events(content, &config()).unwrap_err(); + assert!(err.to_string().contains("VCALENDAR structure")); + } + } + + #[test] + fn duration_timestamp_overflow_skips_only_that_event() { + let content = calendar( + "BEGIN:VEVENT\nSUMMARY:Overflow\nDTSTART:20260306T123000Z\nDURATION:PT9223372036854775807S\nEND:VEVENT\nBEGIN:VEVENT\nSUMMARY:Good\nDTSTART:20260307T123000Z\nDTEND:20260307T133000Z\nEND:VEVENT", + ); + let events = parse_ics_events(&content, &config()).expect("events"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary.as_deref(), Some("Good")); + } +} diff --git a/backend/src/processing/parser/ics/mod.rs b/backend/src/processing/parser/ics/mod.rs new file mode 100644 index 000000000..665fa22a3 --- /dev/null +++ b/backend/src/processing/parser/ics/mod.rs @@ -0,0 +1,274 @@ +mod dummy; +mod event; +mod time; + +use crate::{model::IcsEpgSourceConfig, utils::compressed_file_reader_async::CompressedFileReaderAsync}; +pub use dummy::fill_dummy_gaps; +pub use event::IcsEvent; +use shared::{ + defaults::{MAX_ICS_DESCRIPTION_LENGTH, MAX_ICS_SUMMARY_LENGTH}, + error::TuliproxError, + model::{EpgChannel, EpgProgramme}, + utils::Internable, +}; +use std::{path::Path, sync::Arc}; +use tokio::io::AsyncReadExt; + +pub async fn parse_ics_file_to_channel( + file_path: &Path, + channel_id: Arc, + channel_title: Option>, + config: &IcsEpgSourceConfig, +) -> Result { + let content = read_ics_file_limited(file_path, config.max_decompressed_bytes).await?; + let events = event::parse_ics_events(&content, config)?; + let mut programmes = events_to_programmes(events, &channel_id, config); + programmes.sort_by_key(|programme| (programme.start, programme.stop)); + let channel_title = Some(channel_title.unwrap_or_else(|| Arc::clone(&channel_id))); + + Ok(EpgChannel { id: channel_id, title: channel_title, icon: None, programmes }) +} + +async fn read_ics_file_limited(file_path: &Path, max_bytes: usize) -> Result { + let mut reader = CompressedFileReaderAsync::new(file_path) + .await + .map_err(|err| TuliproxError::Io(format!("Failed to open ICS file {}: {err}", file_path.display())))?; + let mut buffer = Vec::new(); + let mut chunk = [0_u8; 8192]; + + loop { + let read = reader + .read(&mut chunk) + .await + .map_err(|err| TuliproxError::Io(format!("Failed to read ICS file {}: {err}", file_path.display())))?; + if read == 0 { + break; + } + + buffer.extend_from_slice(&chunk[..read]); + if buffer.len() > max_bytes { + return Err(TuliproxError::Parse(format!( + "ICS file {} exceeds max_decompressed_bytes={max_bytes}", + file_path.display() + ))); + } + } + + String::from_utf8(buffer) + .map_err(|err| TuliproxError::Parse(format!("ICS file {} is not valid UTF-8: {err}", file_path.display()))) +} + +fn events_to_programmes( + events: Vec, + channel_id: &Arc, + config: &IcsEpgSourceConfig, +) -> Vec { + let mut result = Vec::with_capacity(events.len()); + + for event in events { + if event.cancelled && !config.include_cancelled { + continue; + } + + let Some(start) = event.start else { + continue; + }; + let Some(stop) = event.stop else { + continue; + }; + if start >= stop { + continue; + } + + let title = render_event_template(&config.event.title, &event, MAX_ICS_SUMMARY_LENGTH); + let mut desc = render_event_template(&config.event.description, &event, MAX_ICS_DESCRIPTION_LENGTH); + + if config.event.include_location { + if let Some(location) = event.location.as_deref().filter(|value| !value.is_empty()) { + append_description_metadata(&mut desc, "Location: ", location); + } + } + + if config.event.include_categories { + if let Some(categories) = event.categories.as_deref().filter(|value| !value.is_empty()) { + append_description_metadata(&mut desc, "Categories: ", categories); + } + } + + result.push(EpgProgramme::new_all( + start, + stop, + Arc::clone(channel_id), + (!title.is_empty()).then(|| title.intern()), + (!desc.is_empty()).then(|| desc.intern()), + None, + )); + } + + result +} + +fn render_event_template(template: &str, event: &IcsEvent, max_bytes: usize) -> String { + let replacements = [ + ("{summary}", event.summary.as_deref().unwrap_or_default()), + ("{description}", event.description.as_deref().unwrap_or_default()), + ("{location}", event.location.as_deref().unwrap_or_default()), + ("{categories}", event.categories.as_deref().unwrap_or_default()), + ("{uid}", event.uid.as_deref().unwrap_or_default()), + ("{start}", event.start_display.as_deref().unwrap_or_default()), + ("{end}", event.stop_display.as_deref().unwrap_or_default()), + ]; + let mut rendered = String::with_capacity(template.len().min(max_bytes)); + let mut remaining = template; + + while !remaining.is_empty() && rendered.len() < max_bytes { + let Some(open_brace) = remaining.find('{') else { + append_utf8_bounded(&mut rendered, remaining, max_bytes); + break; + }; + append_utf8_bounded(&mut rendered, &remaining[..open_brace], max_bytes); + remaining = &remaining[open_brace..]; + + if let Some((placeholder, value)) = + replacements.iter().find(|(placeholder, _)| remaining.starts_with(*placeholder)) + { + append_utf8_bounded(&mut rendered, value, max_bytes); + remaining = &remaining[placeholder.len()..]; + } else { + append_utf8_bounded(&mut rendered, "{", max_bytes); + remaining = &remaining[1..]; + } + } + + rendered +} + +fn append_description_metadata(description: &mut String, label: &str, value: &str) { + if !description.is_empty() { + append_utf8_bounded(description, "\n", MAX_ICS_DESCRIPTION_LENGTH); + } + append_utf8_bounded(description, label, MAX_ICS_DESCRIPTION_LENGTH); + append_utf8_bounded(description, value, MAX_ICS_DESCRIPTION_LENGTH); +} + +fn append_utf8_bounded(target: &mut String, value: &str, max_bytes: usize) { + let available = max_bytes.saturating_sub(target.len()); + let mut end = value.len().min(available); + while !value.is_char_boundary(end) { + end -= 1; + } + target.push_str(&value[..end]); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{IcsEpgSourceConfig, IcsEventMapping}; + use std::fs; + use tempfile::tempdir; + + #[tokio::test] + async fn parses_file_to_channel_and_programmes() { + let dir = tempdir().expect("temp dir"); + let path = dir.path().join("calendar.ics"); + fs::write( + &path, + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nUID:1\nSUMMARY:Practice 1\nDESCRIPTION:Session\nLOCATION:Bahrain\nCATEGORIES:F1,Race\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .expect("write"); + let config = IcsEpgSourceConfig { + event: IcsEventMapping { + title: "Formula 1: {summary}".to_string(), + description: "{description}".to_string(), + include_location: true, + include_categories: true, + }, + ..IcsEpgSourceConfig::default() + }; + + let channel = parse_ics_file_to_channel(&path, "f1.calendar".intern(), Some("Formula 1".intern()), &config) + .await + .expect("channel"); + + assert_eq!(channel.id.as_ref(), "f1.calendar"); + assert_eq!(channel.title.as_deref(), Some("Formula 1")); + assert_eq!(channel.programmes.len(), 1); + assert_eq!(channel.programmes[0].title.as_deref(), Some("Formula 1: Practice 1")); + assert_eq!(channel.programmes[0].desc.as_deref(), Some("Session\nLocation: Bahrain\nCategories: F1,Race")); + } + + #[tokio::test] + async fn cancelled_events_are_skipped_by_default_and_imported_when_enabled() { + let dir = tempdir().expect("temp dir"); + let path = dir.path().join("cancelled.ics"); + fs::write( + &path, + "BEGIN:VCALENDAR\nBEGIN:VEVENT\nSUMMARY:Cancelled\nSTATUS:CANCELLED\nDTSTART:20260306T123000Z\nDTEND:20260306T133000Z\nEND:VEVENT\nEND:VCALENDAR", + ) + .expect("write"); + + let skipped = parse_ics_file_to_channel(&path, "f1.calendar".intern(), None, &IcsEpgSourceConfig::default()) + .await + .expect("channel"); + assert!(skipped.programmes.is_empty()); + + let included_config = IcsEpgSourceConfig { include_cancelled: true, ..IcsEpgSourceConfig::default() }; + let included = + parse_ics_file_to_channel(&path, "f1.calendar".intern(), None, &included_config).await.expect("channel"); + assert_eq!(included.programmes.len(), 1); + assert_eq!(included.title.as_deref(), Some("f1.calendar")); + } + + #[tokio::test] + async fn rejects_file_above_decompressed_limit() { + let dir = tempdir().expect("temp dir"); + let path = dir.path().join("large.ics"); + fs::write(&path, "BEGIN:VCALENDAR\nEND:VCALENDAR\n").expect("write"); + let config = IcsEpgSourceConfig { max_decompressed_bytes: 4, ..IcsEpgSourceConfig::default() }; + + let err = parse_ics_file_to_channel(&path, "f1.calendar".intern(), None, &config).await.unwrap_err(); + assert!(err.to_string().contains("max_decompressed_bytes")); + } + + #[test] + fn bounded_renderer_caps_repeated_placeholders_uid_and_multibyte_text() { + let event = IcsEvent { + uid: Some("u".repeat(MAX_ICS_SUMMARY_LENGTH)), + description: Some("€".repeat(MAX_ICS_DESCRIPTION_LENGTH / 3 + 1)), + ..IcsEvent::default() + }; + let repeated_description = "{description}".repeat(MAX_ICS_SUMMARY_LENGTH / "{description}".len()); + let rendered_description = render_event_template(&repeated_description, &event, MAX_ICS_SUMMARY_LENGTH); + assert!(rendered_description.len() <= MAX_ICS_SUMMARY_LENGTH); + assert!(rendered_description.chars().all(|character| character == '€')); + + let rendered_uid = render_event_template("{uid}{uid}", &event, MAX_ICS_SUMMARY_LENGTH); + assert_eq!(rendered_uid.len(), MAX_ICS_SUMMARY_LENGTH); + } + + #[test] + fn final_description_cap_includes_location_and_categories() { + let event = IcsEvent { + start: Some(1), + stop: Some(2), + description: Some("Description".to_string()), + location: Some("L".repeat(MAX_ICS_DESCRIPTION_LENGTH / 2)), + categories: Some("ä".repeat(MAX_ICS_DESCRIPTION_LENGTH)), + ..IcsEvent::default() + }; + let config = IcsEpgSourceConfig { + event: IcsEventMapping { + title: "{summary}".to_string(), + description: "{description}".to_string(), + include_location: true, + include_categories: true, + }, + ..IcsEpgSourceConfig::default() + }; + + let programmes = events_to_programmes(vec![event], &"channel".intern(), &config); + let description = programmes[0].desc.as_deref().expect("description"); + assert!(description.len() <= MAX_ICS_DESCRIPTION_LENGTH); + assert!(description.is_char_boundary(description.len())); + } +} diff --git a/backend/src/processing/parser/ics/time.rs b/backend/src/processing/parser/ics/time.rs new file mode 100644 index 000000000..12c36db11 --- /dev/null +++ b/backend/src/processing/parser/ics/time.rs @@ -0,0 +1,189 @@ +use chrono::{NaiveDate, NaiveDateTime, TimeZone, Utc}; +use chrono_tz::{Tz, UTC}; +use shared::error::TuliproxError; +use std::collections::HashMap; + +pub struct ParsedIcsDateTime { + pub timestamp: i64, + pub display: String, + pub display_timezone: Tz, + pub all_day: bool, +} + +pub fn parse_ics_datetime( + params: &HashMap, + value: &str, + fallback_timezone: &str, +) -> Result { + if params.get("VALUE").is_some_and(|v| v.eq_ignore_ascii_case("DATE")) { + return parse_ics_date(value, fallback_timezone); + } + + if let Some(stripped) = value.strip_suffix('Z') { + let naive = parse_naive_datetime(stripped)?; + let dt = Utc.from_utc_datetime(&naive); + return Ok(ParsedIcsDateTime { + timestamp: dt.timestamp(), + display: dt.to_rfc3339(), + display_timezone: UTC, + all_day: false, + }); + } + + let timezone = params.get("TZID").map_or(fallback_timezone, String::as_str); + let tz: Tz = + timezone.parse().map_err(|_| TuliproxError::ConfigEpg(format!("Unknown ICS timezone '{timezone}'")))?; + let naive = parse_naive_datetime(value)?; + let local = tz + .from_local_datetime(&naive) + .single() + .or_else(|| tz.from_local_datetime(&naive).earliest()) + .ok_or_else(|| TuliproxError::ConfigEpg(format!("Invalid local ICS datetime '{value}' in {timezone}")))?; + + Ok(ParsedIcsDateTime { + timestamp: local.with_timezone(&Utc).timestamp(), + display: local.to_rfc3339(), + display_timezone: tz, + all_day: false, + }) +} + +pub fn display_from_timestamp_in_timezone(timestamp: i64, timezone: Tz) -> Option { + timezone.timestamp_opt(timestamp, 0).single().map(|dt| dt.to_rfc3339()) +} + +fn parse_ics_date(value: &str, fallback_timezone: &str) -> Result { + let tz: Tz = fallback_timezone + .parse() + .map_err(|_| TuliproxError::ConfigEpg(format!("Unknown ICS timezone '{fallback_timezone}'")))?; + let date = NaiveDate::parse_from_str(value, "%Y%m%d") + .map_err(|err| TuliproxError::ConfigEpg(format!("Invalid ICS DATE '{value}': {err}")))?; + let naive = + date.and_hms_opt(0, 0, 0).ok_or_else(|| TuliproxError::ConfigEpg(format!("Invalid ICS DATE '{value}'")))?; + let local = + tz.from_local_datetime(&naive).single().or_else(|| tz.from_local_datetime(&naive).earliest()).ok_or_else( + || TuliproxError::ConfigEpg(format!("Invalid local ICS DATE '{value}' in {fallback_timezone}")), + )?; + + Ok(ParsedIcsDateTime { + timestamp: local.with_timezone(&Utc).timestamp(), + display: local.to_rfc3339(), + display_timezone: tz, + all_day: true, + }) +} + +fn parse_naive_datetime(value: &str) -> Result { + NaiveDateTime::parse_from_str(value, "%Y%m%dT%H%M%S") + .or_else(|_| NaiveDateTime::parse_from_str(value, "%Y%m%dT%H%M")) + .map_err(|err| TuliproxError::ConfigEpg(format!("Invalid ICS datetime '{value}': {err}"))) +} + +pub fn parse_ics_duration(value: &str) -> Option { + let value = value.strip_prefix('P')?; + if value.is_empty() { + return None; + } + + let mut in_time = false; + let mut saw_time_designator = false; + let mut saw_time_component = false; + let mut saw_component = false; + let mut last_component_order = 0_u8; + let mut number: Option = None; + let mut seconds = 0_i64; + + for ch in value.chars() { + if ch.is_ascii_digit() { + let digit = i64::from(ch.to_digit(10)?); + number = Some(number.unwrap_or_default().checked_mul(10)?.checked_add(digit)?); + continue; + } + + if ch == 'T' { + if in_time || number.is_some() { + return None; + } + in_time = true; + saw_time_designator = true; + continue; + } + + let (component_order, multiplier) = match ch { + 'D' if !in_time => (1, 86_400_i64), + 'H' if in_time => (2, 3_600_i64), + 'M' if in_time => (3, 60_i64), + 'S' if in_time => (4, 1_i64), + _ => return None, + }; + if component_order <= last_component_order { + return None; + } + + let component = number.take()?.checked_mul(multiplier)?; + seconds = seconds.checked_add(component)?; + last_component_order = component_order; + saw_component = true; + saw_time_component |= in_time; + } + + (number.is_none() && saw_component && (!saw_time_designator || saw_time_component) && seconds > 0) + .then_some(seconds) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn params(items: &[(&str, &str)]) -> HashMap { + items.iter().map(|(key, value)| ((*key).to_string(), (*value).to_string())).collect() + } + + #[test] + fn parses_utc_datetime() { + let parsed = parse_ics_datetime(&HashMap::new(), "20260306T123000Z", "Europe/Berlin").expect("datetime"); + assert_eq!(parsed.timestamp, 1_772_800_200); + assert!(!parsed.all_day); + } + + #[test] + fn parses_timezone_datetime() { + let parsed = + parse_ics_datetime(¶ms(&[("TZID", "Europe/Berlin")]), "20260306T123000", "UTC").expect("datetime"); + assert_eq!(parsed.timestamp, 1_772_796_600); + assert_eq!(parsed.display, "2026-03-06T12:30:00+01:00"); + } + + #[test] + fn parses_floating_datetime_with_fallback_timezone() { + let parsed = parse_ics_datetime(&HashMap::new(), "20260306T123000", "Europe/Berlin").expect("datetime"); + assert_eq!(parsed.timestamp, 1_772_796_600); + } + + #[test] + fn parses_duration() { + assert_eq!(parse_ics_duration("PT1H30M"), Some(5_400)); + assert_eq!(parse_ics_duration("P1DT2H"), Some(93_600)); + assert_eq!(parse_ics_duration("-PT1H"), None); + } + + #[test] + fn rejects_incomplete_and_non_positive_durations() { + for duration in ["PT1H2", "P1D2", "PT", "P", "PT0S", "-PT1H"] { + assert_eq!(parse_ics_duration(duration), None, "duration {duration}"); + } + } + + #[test] + fn rejects_duration_component_and_sum_overflow() { + assert_eq!(parse_ics_duration(&format!("P{}D", i64::MAX)), None); + assert_eq!(parse_ics_duration(&format!("P1DT{}S", i64::MAX)), None); + assert_eq!(parse_ics_duration("PT999999999999999999999999999999999999S"), None); + } + + #[test] + fn marks_value_date_as_all_day() { + let parsed = parse_ics_datetime(¶ms(&[("VALUE", "DATE")]), "20260306", "UTC").expect("date"); + assert!(parsed.all_day); + } +} diff --git a/backend/src/processing/parser/mod.rs b/backend/src/processing/parser/mod.rs index 5cfbaf9b8..1f03882d5 100644 --- a/backend/src/processing/parser/mod.rs +++ b/backend/src/processing/parser/mod.rs @@ -1,4 +1,5 @@ pub mod m3u; pub mod xtream; pub mod xmltv; -pub mod hls; \ No newline at end of file +pub mod hls; +pub mod ics; diff --git a/backend/src/processing/parser/xmltv.rs b/backend/src/processing/parser/xmltv.rs index 9ef8fcfd4..813ced7c0 100644 --- a/backend/src/processing/parser/xmltv.rs +++ b/backend/src/processing/parser/xmltv.rs @@ -1,23 +1,38 @@ -use crate::model::{ - Epg, TVGuide, XmlTag, XmlTagIcon, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_CHANNEL, EPG_TAG_DISPLAY_NAME, - EPG_TAG_ICON, EPG_TAG_PROGRAMME, EPG_TAG_TV, +use crate::{ + model::{ + Epg, EpgSmartMatchConfig, IcsDummyPolicy, IcsEpgSourceConfig, PersistedEpgSource, PersistedEpgSourceKind, + TVGuide, XmlTag, XmlTagIcon, EPG_ATTRIB_CHANNEL, EPG_ATTRIB_ID, EPG_TAG_CHANNEL, EPG_TAG_DISPLAY_NAME, + EPG_TAG_ICON, EPG_TAG_PROGRAMME, EPG_TAG_TV, + }, + processing::{ + parser::ics, + processor::{with_folded_epg_id, EpgIdCache}, + }, + utils::{async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, parse_xmltv_time}, }; -use crate::model::{EpgSmartMatchConfig, PersistedEpgSource}; -use crate::processing::processor::{with_folded_epg_id, EpgIdCache}; -use crate::utils::compressed_file_reader_async::CompressedFileReaderAsync; -use crate::utils::{async_file_reader, parse_xmltv_time}; use log::error; use quick_xml::events::{BytesStart, BytesText, Event}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use shared::concat_string; -use shared::model::{EpgChannel, EpgNamePrefix, EpgProgramme}; -use shared::utils::{deunicode_string, Internable, CONSTANTS}; -use std::borrow::Cow; -use std::cmp::min; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use shared::{ + concat_string, + model::{EpgChannel, EpgNamePrefix, EpgProgramme}, + utils::{deunicode_string, Internable, CONSTANTS}, +}; +use std::{ + borrow::Cow, + cmp::min, + collections::{HashMap, HashSet}, + sync::Arc, +}; use tokio::io::AsyncRead; +struct IcsPersistedSource<'a> { + channel_id: &'a Arc, + channel_title: Option<&'a Arc>, + match_names: &'a [Arc], + config: &'a IcsEpgSourceConfig, +} + /// Splits a string at the first delimiter if the prefix matches a known country code. /// /// Returns a tuple containing the country code prefix (if found) and the remainder of the string, both trimmed. If no valid prefix is found, returns `None` and the original input. @@ -52,7 +67,6 @@ fn split_by_first_match<'a>(input: &'a str, delimiters: &[char]) -> (Option<&'a (None, input) } - fn name_prefix<'a>(name: &'a str, smart_config: &EpgSmartMatchConfig) -> (&'a str, Option<&'a str>) { if smart_config.name_prefix != EpgNamePrefix::Ignore { let (prefix, suffix) = split_by_first_match(name, &smart_config.name_prefix_separator); @@ -78,32 +92,23 @@ pub fn normalize_channel_name(name: &str, normalize_config: &EpgSmartMatchConfig // Remove all non-alphanumeric characters (except dashes and underscores). let cleaned_name = normalize_config.normalize_regex.replace_all(channel_name, ""); // Remove terms like resolution - let cleaned_name = normalize_config.strip.iter().fold(cleaned_name.to_string(), |acc, term| { - acc.replace(term, "") - }); + let cleaned_name = normalize_config.strip.iter().fold(cleaned_name.to_string(), |acc, term| acc.replace(term, "")); match suffix { None => cleaned_name, - Some(sfx) => { - match &normalize_config.name_prefix { - EpgNamePrefix::Ignore => cleaned_name, - EpgNamePrefix::Suffix(sep) => combine(sep, &cleaned_name, sfx), - EpgNamePrefix::Prefix(sep) => combine(sep, sfx, &cleaned_name), - } - } + Some(sfx) => match &normalize_config.name_prefix { + EpgNamePrefix::Ignore => cleaned_name, + EpgNamePrefix::Suffix(sep) => combine(sep, &cleaned_name, sfx), + EpgNamePrefix::Prefix(sep) => combine(sep, sfx, &cleaned_name), + }, } } - impl TVGuide { - pub fn merge(epgs: Vec) -> Option { - flatten_tvguide(epgs) - } + pub fn merge(epgs: Vec) -> Option { flatten_tvguide(epgs) } fn prepare_tag(id_cache: &mut EpgIdCache, tag: &mut XmlTag, smart_match: bool) { { - let maybe_epg_id = { - tag.get_attribute_value(&EPG_ATTRIB_ID.intern()).cloned() - }; + let maybe_epg_id = { tag.get_attribute_value(&EPG_ATTRIB_ID.intern()).cloned() }; if let Some(epg_id) = maybe_epg_id { tag.normalized_epg_ids .get_or_insert_with(Vec::new) @@ -116,11 +121,11 @@ impl TVGuide { for child in children { match child.name.as_ref() { EPG_TAG_DISPLAY_NAME if smart_match => { - if let Some(name) = &child.value { - tag.normalized_epg_ids - .get_or_insert_with(Vec::new) - .push(normalize_channel_name(name, &id_cache.smart_match_config).intern()); - } + if let Some(name) = &child.value { + tag.normalized_epg_ids + .get_or_insert_with(Vec::new) + .push(normalize_channel_name(name, &id_cache.smart_match_config).intern()); + } } EPG_TAG_ICON => { if let Some(src) = child.get_attribute_value(&src) { @@ -138,10 +143,8 @@ impl TVGuide { } fn try_fuzzy_matching(id_cache: &mut EpgIdCache, epg_id: &Arc, tag: &XmlTag, fuzzy_matching: bool) -> bool { - let mut matched = tag - .normalized_epg_ids - .as_ref() - .is_some_and(|ids| id_cache.match_with_normalized(epg_id, ids)); + let mut matched = + tag.normalized_epg_ids.as_ref().is_some_and(|ids| id_cache.match_with_normalized(epg_id, ids)); if !matched && fuzzy_matching { let (fuzzy_matched, matched_normalized_name) = Self::find_best_fuzzy_match(id_cache, tag); if fuzzy_matched { @@ -184,7 +187,9 @@ impl TVGuide { tag_title: &Arc, tag_desc: &Arc, ) -> Option { - let Some((Some(start), Some(stop))) = tag.attributes.as_ref().map(|a| (a.get(start_attrib), a.get(stop_attrib))) else { + let Some((Some(start), Some(stop))) = + tag.attributes.as_ref().map(|a| (a.get(start_attrib), a.get(stop_attrib))) + else { error!("Missing start or stop attribute in programme tag, skipping"); return None; }; @@ -208,14 +213,7 @@ impl TVGuide { let catchup_id = tag.attributes.as_ref().and_then(|attributes| attributes.get(catchup_id_attrib)).cloned(); - Some(EpgProgramme::new_all( - start_time, - stop_time, - Arc::clone(epg_id), - title, - desc, - catchup_id, - )) + Some(EpgProgramme::new_all(start_time, stop_time, Arc::clone(epg_id), title, desc, catchup_id)) } /// Finds the best fuzzy match for a channel's normalized EPG ID using phonetic encoding and Jaro-Winkler similarity. @@ -245,10 +243,8 @@ impl TVGuide { }; // 1) Precalculation: (tag_normalized, tag_code) - let pre: Vec<(Arc, Arc)> = normalized_epg_ids - .iter() - .map(|tn| (tn.clone(), id_cache.phonetic(tn))) - .collect(); + let pre: Vec<(Arc, Arc)> = + normalized_epg_ids.iter().map(|tn| (tn.clone(), id_cache.phonetic(tn))).collect(); // 2) Early exit if match >= best_match_threshold for (tag_normalized, tag_code) in &pre { @@ -329,7 +325,8 @@ impl TVGuide { let mut filter_tags = |mut tag: XmlTag| { match tag.name.as_ref() { EPG_TAG_CHANNEL => { - let tag_epg_id = tag.get_attribute_value(&epg_attrib_id).map_or_else(|| "".intern(), Internable::intern); + let tag_epg_id = + tag.get_attribute_value(&epg_attrib_id).map_or_else(|| "".intern(), Internable::intern); if tag_epg_id.is_empty() { return; } @@ -364,17 +361,15 @@ impl TVGuide { EPG_TAG_PROGRAMME => { if let Some(epg_id) = tag.get_attribute_value(&epg_attrib_channel) { if with_folded_epg_id(epg_id, |folded| source_processed.contains(folded)) { - if let Some(programme) = - Self::extract_programme( - &tag, - epg_id, - &start_attrib, - &stop_attrib, - &catchup_id_attrib, - &tag_title, - &tag_desc, - ) - { + if let Some(programme) = Self::extract_programme( + &tag, + epg_id, + &start_attrib, + &stop_attrib, + &catchup_id_attrib, + &tag_title, + &tag_desc, + ) { accumulator.push_programme(epg_source.priority, source_order, programme); } } @@ -397,6 +392,64 @@ impl TVGuide { } } + async fn process_ics_file( + id_cache: &mut EpgIdCache, + epg_source: &PersistedEpgSource, + source_order: usize, + accumulator: &mut EpgMergeAccumulator, + source: IcsPersistedSource<'_>, + ) -> bool { + let IcsPersistedSource { channel_id, channel_title, match_names, config } = source; + let mut candidates = Vec::with_capacity(2 + match_names.len()); + candidates.push(channel_id.to_string()); + if let Some(title) = channel_title { + candidates.push(title.to_string()); + } + candidates.extend(match_names.iter().map(ToString::to_string)); + let normalized_candidates = id_cache.normalize_candidates(candidates); + + let add_channel = if id_cache.smart_match_enabled { + id_cache.contains_channel_epg_id(channel_id) + || id_cache.match_epg_channel_candidates(channel_id, &normalized_candidates) + } else { + id_cache.contains_channel_epg_id(channel_id) + }; + + if !add_channel { + return false; + } + + match ics::parse_ics_file_to_channel( + &epg_source.file_path, + Arc::clone(channel_id), + channel_title.cloned(), + config, + ) + .await + { + Ok(channel) => { + id_cache.insert_processed_epg_id(channel_id); + accumulator.add_channel_with_programmes( + epg_source.priority, + source_order, + epg_source.logo_override, + channel, + ); + accumulator.register_dummy_policy( + channel_id, + epg_source.priority, + source_order, + IcsDummyPolicy { timezone: config.timezone.clone(), config: config.dummy.clone() }, + ); + true + } + Err(err) => { + log::warn!("Failed to process ICS EPG file {}: {err}", epg_source.file_path.display()); + false + } + } + } + pub async fn filter(&self, id_cache: &mut EpgIdCache) -> Option> { self.filter_merged(id_cache).await.map(|epg| vec![epg]) } @@ -414,7 +467,30 @@ impl TVGuide { } let mut accumulator = EpgMergeAccumulator::new(); for (source_order, epg_source) in self.get_epg_sources().iter().enumerate() { - Self::process_epg_file(id_cache, epg_source, source_order, &mut accumulator).await; + let _source_read_lock = match self.get_file_locks() { + Some(file_locks) => Some(file_locks.read_lock(&epg_source.file_path).await), + None => None, + }; + match &epg_source.kind { + PersistedEpgSourceKind::Xmltv => { + Self::process_epg_file(id_cache, epg_source, source_order, &mut accumulator).await; + } + PersistedEpgSourceKind::Ics { channel_id, channel_title, match_names, config } => { + Self::process_ics_file( + id_cache, + epg_source, + source_order, + &mut accumulator, + IcsPersistedSource { + channel_id, + channel_title: channel_title.as_ref(), + match_names, + config: config.as_ref(), + }, + ) + .await; + } + } } accumulator.finish_epg_with_icon_overrides() } @@ -524,9 +600,7 @@ enum XmlTagType { impl XmlTagType { #[inline] - pub(crate) fn is_tv(self) -> bool { - self == XmlTagType::Tv - } + pub(crate) fn is_tv(self) -> bool { self == XmlTagType::Tv } } fn get_tag_type(name: &str) -> XmlTagType { @@ -534,12 +608,14 @@ fn get_tag_type(name: &str) -> XmlTagType { EPG_TAG_TV => XmlTagType::Tv, EPG_TAG_CHANNEL => XmlTagType::Channel, EPG_TAG_PROGRAMME => XmlTagType::Programme, - _ => XmlTagType::Ignored + _ => XmlTagType::Ignored, } } fn collect_tag_attributes(e: &BytesStart) -> HashMap, Arc> { - let attributes = e.attributes().filter_map(Result::ok) + let attributes = e + .attributes() + .filter_map(Result::ok) .filter_map(|a| { let key_binding = a.key; let key_raw = String::from_utf8_lossy(key_binding.as_ref()); @@ -556,7 +632,8 @@ fn collect_tag_attributes(e: &BytesStart) -> HashMap, Arc> { } else { None } - }).collect::, Arc>>(); + }) + .collect::, Arc>>(); attributes } @@ -567,6 +644,22 @@ struct PreferredAttributes { attributes: HashMap, Arc>, } +#[derive(Debug)] +struct PreferredDummyPolicy { + priority: i16, + source_order: usize, + policy: IcsDummyPolicy, +} + +/// Carries the source rank required to select a preview dummy policy exactly like the main EPG merge. +#[derive(Debug)] +pub(crate) struct EpgDummyPolicySource { + pub priority: i16, + pub source_order: usize, + pub channel_id: Arc, + pub policy: IcsDummyPolicy, +} + type FinishedEpgChannels = (Option, Arc>>, Vec); pub(crate) type MergedEpgWithIconOverrides = (Epg, HashSet>); @@ -592,6 +685,7 @@ struct ChannelMergeAcc { struct EpgMergeAccumulator { attributes: Option, channels: HashMap, ChannelMergeAcc>, + dummy_policies: HashMap, PreferredDummyPolicy>, } impl EpgMergeAccumulator { @@ -661,13 +755,35 @@ impl EpgMergeAccumulator { } fn push_programme(&mut self, priority: i16, source_order: usize, programme: EpgProgramme) { - if let Some(channel) = with_folded_epg_id(programme.get_transient_channel_id(), |folded| self.channels.get_mut(folded)) { + if let Some(channel) = + with_folded_epg_id(programme.get_transient_channel_id(), |folded| self.channels.get_mut(folded)) + { channel.programmes.push(ProgrammeMergeEntry { priority, source_order, programme }); } else { error!("Channel {} not found in EPG, dangling programme", programme.get_transient_channel_id()); } } + fn register_dummy_policy( + &mut self, + channel_id: &Arc, + priority: i16, + source_order: usize, + policy: IcsDummyPolicy, + ) { + if !policy.config.enabled { + return; + } + let key = with_folded_epg_id(channel_id, |folded| folded.intern()); + let replace = self + .dummy_policies + .get(&key) + .is_none_or(|current| (priority, source_order) < (current.priority, current.source_order)); + if replace { + self.dummy_policies.insert(key, PreferredDummyPolicy { priority, source_order, policy }); + } + } + fn add_channel_with_programmes( &mut self, priority: i16, @@ -682,8 +798,11 @@ impl EpgMergeAccumulator { if !acc.programmes.is_empty() { acc.needs_programme_merge = true; } - acc.programmes - .extend(programmes.into_iter().map(|programme| ProgrammeMergeEntry { priority, source_order, programme })); + acc.programmes.extend(programmes.into_iter().map(|programme| ProgrammeMergeEntry { + priority, + source_order, + programme, + })); } } @@ -716,7 +835,7 @@ impl EpgMergeAccumulator { } fn finish_epg_with_icon_overrides(self) -> Option { - let EpgMergeAccumulator { attributes, channels } = self; + let EpgMergeAccumulator { attributes, channels, dummy_policies } = self; let mut channels = channels.into_values().collect::>(); if channels.is_empty() { return None; @@ -726,6 +845,8 @@ impl EpgMergeAccumulator { normalize_channel_programmes(acc); } + apply_dummy_policies(&mut channels, &dummy_policies); + channels.sort_by(|left, right| left.channel.id.cmp(&right.channel.id)); let icon_override_channels = channels .iter() @@ -746,6 +867,25 @@ impl EpgMergeAccumulator { } } +fn apply_dummy_policies(channels: &mut [ChannelMergeAcc], dummy_policies: &HashMap, PreferredDummyPolicy>) { + let now = chrono::Utc::now(); + for acc in channels { + let key = with_folded_epg_id(&acc.channel.id, |folded| folded.intern()); + if let Some(preferred) = dummy_policies.get(&key) { + let policy = &preferred.policy; + if let Err(err) = ics::fill_dummy_gaps( + &mut acc.channel.programmes, + &acc.channel.id, + &policy.timezone, + &policy.config, + now, + ) { + log::warn!("Failed to apply ICS dummy policy for {}: {err}", acc.channel.id); + } + } + } +} + fn backfill_programme_metadata(existing: &mut EpgProgramme, incoming: EpgProgramme) { if existing.title.is_none() { existing.title = incoming.title; @@ -761,14 +901,8 @@ fn backfill_programme_metadata(existing: &mut EpgProgramme, incoming: EpgProgram fn normalize_channel_programmes(acc: &mut ChannelMergeAcc) { // Always normalize programme order and dedupe, even for single-source channels. // This preserves backfill behavior for duplicate entries within the same source. - acc.programmes.sort_by_key(|entry| { - ( - entry.programme.start, - entry.programme.stop, - entry.priority, - entry.source_order, - ) - }); + acc.programmes + .sort_by_key(|entry| (entry.programme.start, entry.programme.stop, entry.priority, entry.source_order)); let programme_entries = std::mem::take(&mut acc.programmes); let mut merged_programmes = Vec::with_capacity(programme_entries.len()); @@ -790,6 +924,7 @@ fn normalize_channel_programmes(acc: &mut ChannelMergeAcc) { acc.channel.programmes = merged_programmes; } +#[cfg(test)] pub(crate) fn merge_epg_channels_by_priority(channels_by_source: Vec<(i16, Vec)>) -> Vec { let mut accumulator = EpgMergeAccumulator::new(); for (source_order, (priority, channels)) in channels_by_source.into_iter().enumerate() { @@ -800,6 +935,25 @@ pub(crate) fn merge_epg_channels_by_priority(channels_by_source: Vec<(i16, Vec)>, + dummy_policies: Vec, +) -> Vec { + let mut accumulator = EpgMergeAccumulator::new(); + for (source_order, (priority, channels)) in channels_by_source.into_iter().enumerate() { + for channel in channels { + accumulator.add_channel_with_programmes(priority, source_order, false, channel); + } + } + for source in dummy_policies { + accumulator.register_dummy_policy(&source.channel_id, source.priority, source.source_order, source.policy); + } + accumulator + .finish_epg_with_icon_overrides() + .map(|(epg, _)| epg.children.into_iter().map(Arc::unwrap_or_clone).collect()) + .unwrap_or_default() +} + pub fn flatten_tvguide(tv_guides: Vec) -> Option { let mut accumulator = EpgMergeAccumulator::new(); for (source_order, guide) in tv_guides.into_iter().enumerate() { @@ -818,15 +972,19 @@ pub fn flatten_tvguide(tv_guides: Vec) -> Option { #[cfg(test)] mod tests { - use crate::model::{Epg, EpgSmartMatchConfig, PersistedEpgSource, TVGuide}; - use crate::processing::parser::xmltv::{ - flatten_tvguide, merge_epg_channels_by_priority, normalize_channel_name, EpgMergeAccumulator, + use crate::{ + model::{ + Epg, EpgSmartMatchConfig, IcsDummyConfig, IcsEpgSourceConfig, PersistedEpgSource, PersistedEpgSourceKind, + TVGuide, + }, + processing::parser::xmltv::{ + flatten_tvguide, merge_epg_channels_by_priority, merge_epg_channels_by_priority_with_dummy_policies, + normalize_channel_name, EpgDummyPolicySource, EpgMergeAccumulator, + }, + utils::FileLockManager, }; use shared::model::{EpgChannel, EpgProgramme}; - use std::collections::HashSet; - use std::fs; - use std::path::PathBuf; - use std::sync::Arc; + use std::{collections::HashSet, fs, path::PathBuf, sync::Arc}; use tempfile::tempdir; /// Run an async test body on a freshly-created multi-threaded tokio @@ -839,6 +997,30 @@ mod tests { tokio::runtime::Runtime::new().unwrap().block_on(future); } + fn xmltv_source(file_path: PathBuf, priority: i16, logo_override: bool) -> PersistedEpgSource { + PersistedEpgSource { file_path, priority, logo_override, kind: PersistedEpgSourceKind::Xmltv } + } + + fn dummy_policy_source(priority: i16, source_order: usize, title: &str) -> EpgDummyPolicySource { + EpgDummyPolicySource { + priority, + source_order, + channel_id: "f1.calendar".intern(), + policy: crate::model::IcsDummyPolicy { + timezone: "UTC".to_string(), + config: IcsDummyConfig { + enabled: true, + title: title.to_string(), + description: String::new(), + days_past: 0, + days_future: 0, + block_hours: 24, + min_gap_minutes: 1, + }, + }, + } + } + #[test] /// Tests normalization of a channel name using the default smart match configuration. /// @@ -1096,16 +1278,9 @@ mod tests { ) .unwrap(); - let mut smart_cfg = EpgSmartMatchConfigDto { - enabled: true, - ..Default::default() - }; + let mut smart_cfg = EpgSmartMatchConfigDto { enabled: true, ..Default::default() }; smart_cfg.prepare().expect("smart match config"); - let tv_guide = TVGuide::new(vec![PersistedEpgSource { - file_path: epg_path, - priority: 0, - logo_override: false, - }]); + let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); let mut id_cache = EpgIdCache::new(Some(&crate::model::EpgConfig { sources: vec![], smart_match: Some(EpgSmartMatchConfig::from(smart_cfg)), @@ -1120,6 +1295,39 @@ mod tests { }); } + #[test] + fn persisted_epg_source_read_uses_shared_file_lock() { + run_async_test(async move { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("locked.xml"); + fs::write( + &epg_path, + r#" + Demo + + Locked read + +"#, + ) + .expect("write XMLTV fixture"); + + let file_locks = Arc::new(FileLockManager::new()); + let guide = TVGuide::new(vec![xmltv_source(epg_path.clone(), 0, false)]) + .with_file_locks(Arc::clone(&file_locks)); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("demo.channel"); + let write_guard = file_locks.write_lock(&epg_path).await; + let filter = guide.filter_merged(&mut id_cache); + tokio::pin!(filter); + + assert!(tokio::time::timeout(std::time::Duration::from_millis(25), filter.as_mut()).await.is_err()); + drop(write_guard); + + let merged = filter.await.expect("EPG parse after write lock release"); + assert_eq!(merged.children[0].programmes[0].title.as_deref(), Some("Locked read")); + }); + } + #[test] fn epg_channel_id_match_is_case_insensitive_and_preserves_guide_case() { run_async_test(async move { @@ -1147,11 +1355,7 @@ mod tests { ) .unwrap(); - let tv_guide = TVGuide::new(vec![PersistedEpgSource { - file_path: epg_path, - priority: 0, - logo_override: false, - }]); + let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); let mut id_cache = EpgIdCache::new(None); // Playlist epg ids arrive in a *different* case than the guide. Both origins // (an Xtream source and a mapper-script literal) go through the same @@ -1193,11 +1397,7 @@ mod tests { ) .unwrap(); - let tv_guide = TVGuide::new(vec![PersistedEpgSource { - file_path: epg_path, - priority: 0, - logo_override: false, - }]); + let tv_guide = TVGuide::new(vec![xmltv_source(epg_path, 0, false)]); let mut id_cache = EpgIdCache::new(None); id_cache.insert_channel_epg_id("demo.channel"); @@ -1244,10 +1444,7 @@ mod tests { ) .unwrap(); - let tv_guide = TVGuide::new(vec![ - PersistedEpgSource { file_path: high_path, priority: 0, logo_override: false }, - PersistedEpgSource { file_path: low_path, priority: 10, logo_override: false }, - ]); + let tv_guide = TVGuide::new(vec![xmltv_source(high_path, 0, false), xmltv_source(low_path, 10, false)]); let mut id_cache = EpgIdCache::new(None); id_cache.insert_channel_epg_id("demo.channel"); @@ -1268,6 +1465,118 @@ mod tests { }); } + #[test] + fn ics_dummy_policy_fills_after_real_programme_merge_without_overwriting_xmltv() { + run_async_test(async move { + use chrono::{Datelike, TimeZone, Utc}; + + let dir = tempdir().unwrap(); + let xml_path = dir.path().join("guide.xml"); + let ics_path = dir.path().join("empty.ics"); + let now = Utc::now(); + let real_start = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 4, 0, 0).single().expect("start"); + let real_stop = Utc.with_ymd_and_hms(now.year(), now.month(), now.day(), 6, 0, 0).single().expect("stop"); + + fs::write( + &xml_path, + format!( + r#" + + + Formula 1 + + + Real Show + +"#, + start = real_start.format("%Y%m%d%H%M%S %z"), + stop = real_stop.format("%Y%m%d%H%M%S %z"), + ), + ) + .unwrap(); + fs::write(&ics_path, "BEGIN:VCALENDAR\nEND:VCALENDAR\n").unwrap(); + + let guide = TVGuide::new(vec![ + xmltv_source(xml_path, 0, false), + PersistedEpgSource { + file_path: ics_path, + priority: 1, + logo_override: false, + kind: PersistedEpgSourceKind::Ics { + channel_id: "f1.calendar".intern(), + channel_title: Some("Formula 1".intern()), + match_names: Vec::new(), + config: Box::new(IcsEpgSourceConfig { + timezone: "UTC".to_string(), + dummy: IcsDummyConfig { + enabled: true, + title: "No programme".to_string(), + description: String::new(), + days_past: 0, + days_future: 0, + block_hours: 4, + min_gap_minutes: 1, + }, + ..IcsEpgSourceConfig::default() + }), + }, + }, + ]); + let mut id_cache = EpgIdCache::new(None); + id_cache.insert_channel_epg_id("f1.calendar"); + + let merged = guide.filter_merged(&mut id_cache).await.expect("merged"); + let channel = &merged.children[0]; + assert_eq!(channel.id.as_ref(), "f1.calendar"); + assert_eq!( + channel.programmes.iter().filter(|programme| programme.title.as_deref() == Some("Real Show")).count(), + 1 + ); + let dummy_programmes = channel + .programmes + .iter() + .filter(|programme| programme.title.as_deref() == Some("No programme")) + .collect::>(); + assert!(!dummy_programmes.is_empty()); + for dummy in dummy_programmes { + assert!( + dummy.stop <= real_start.timestamp() || dummy.start >= real_stop.timestamp(), + "dummy overlaps real programme: {}-{}", + dummy.start, + dummy.stop + ); + } + }); + } + + #[test] + fn dummy_policy_selection_uses_priority_then_source_order() { + let channel = || EpgChannel { + id: "f1.calendar".intern(), + title: Some("Formula 1".intern()), + icon: None, + programmes: Vec::new(), + }; + let merge = |policies| { + merge_epg_channels_by_priority_with_dummy_policies(vec![(0, vec![channel()])], policies) + .into_iter() + .next() + .expect("merged channel") + }; + + let priority_winner = + merge(vec![dummy_policy_source(10, 0, "Low priority"), dummy_policy_source(-10, 1, "High priority")]); + assert!(!priority_winner.programmes.is_empty()); + assert!(priority_winner.programmes.iter().all(|programme| programme.title.as_deref() == Some("High priority"))); + + let source_order_winner = + merge(vec![dummy_policy_source(0, 2, "Later source"), dummy_policy_source(0, 1, "Earlier source")]); + assert!(!source_order_winner.programmes.is_empty()); + assert!(source_order_winner + .programmes + .iter() + .all(|programme| programme.title.as_deref() == Some("Earlier source"))); + } #[ignore = "requires a local XMLTV fixture under /tmp"] #[test] @@ -1277,7 +1586,7 @@ mod tests { let file_path = PathBuf::from("/tmp/invalid_epg.xml"); if file_path.exists() { - let tv_guide = TVGuide::new(vec![PersistedEpgSource { file_path, priority: 0, logo_override: false }]); + let tv_guide = TVGuide::new(vec![xmltv_source(file_path, 0, false)]); let mut id_cache = EpgIdCache::new(None); id_cache.insert_channel_epg_id("342"); @@ -1294,9 +1603,7 @@ mod tests { } } }; - tokio::runtime::Runtime::new() - .unwrap() - .block_on(run_test()); + tokio::runtime::Runtime::new().unwrap().block_on(run_test()); } #[test] @@ -1309,7 +1616,11 @@ mod tests { /// // This will assert that various channel names are normalized as expected. /// ``` fn normalize() { - let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { enabled: true, name_prefix: EpgNamePrefix::Suffix(".".to_string()), ..Default::default() }; + let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { + enabled: true, + name_prefix: EpgNamePrefix::Suffix(".".to_string()), + ..Default::default() + }; let _ = epg_smart_cfg_dto.prepare(); let epg_smart_cfg = EpgSmartMatchConfig::from(epg_smart_cfg_dto); println!("{epg_smart_cfg:?}"); @@ -1323,8 +1634,10 @@ mod tests { use crate::processing::processor::EpgIdCache; use rphonetic::{Encoder, Metaphone}; - use shared::model::{EpgNamePrefix, EpgSmartMatchConfigDto}; - use shared::utils::Internable; + use shared::{ + model::{EpgNamePrefix, EpgSmartMatchConfigDto}, + utils::Internable, + }; #[test] /// Demonstrates phonetic encoding (Metaphone) of normalized channel names with various prefixes and suffixes. @@ -1339,7 +1652,11 @@ mod tests { /// ``` fn test_metaphone() { let metaphone = Metaphone::default(); - let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { enabled: true, name_prefix: EpgNamePrefix::Suffix(".".to_string()), ..Default::default() }; + let mut epg_smart_cfg_dto = EpgSmartMatchConfigDto { + enabled: true, + name_prefix: EpgNamePrefix::Suffix(".".to_string()), + ..Default::default() + }; let _ = epg_smart_cfg_dto.prepare(); let epg_smart_cfg = EpgSmartMatchConfig::from(epg_smart_cfg_dto); println!("{epg_smart_cfg:?}"); diff --git a/backend/src/processing/parser/xtream.rs b/backend/src/processing/parser/xtream.rs index 6312086f7..3f50676db 100644 --- a/backend/src/processing/parser/xtream.rs +++ b/backend/src/processing/parser/xtream.rs @@ -476,6 +476,22 @@ mod tests { } } + #[tokio::test] + async fn xtream_live_stream_preserves_epg_channel_id_for_common_epg_matching() { + let categories = make_reader(r#"[{"category_id":"1","category_name":"Sports"}]"#); + let streams = make_reader( + r#"[{"name":"Formula 1","stream_id":"100","category_id":"1","epg_channel_id":"f1.calendar"}]"#, + ); + + let groups = parse_xtream(&test_input(), XtreamCluster::Live, categories, streams) + .await + .expect("parse xtream") + .expect("groups"); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].channels[0].header.epg_channel_id.as_deref(), Some("f1.calendar")); + } + #[test] fn test_read_json_file_into_struct() { if fs::exists("/tmp/series-info.json").unwrap_or(false) { diff --git a/backend/src/processing/processor/epg.rs b/backend/src/processing/processor/epg.rs index 094027348..aa2992068 100644 --- a/backend/src/processing/processor/epg.rs +++ b/backend/src/processing/processor/epg.rs @@ -202,6 +202,71 @@ impl EpgIdCache { } false } + + pub fn normalize_candidates(&self, candidates: I) -> Vec> + where + I: IntoIterator, + S: AsRef, + { + candidates + .into_iter() + .map(|candidate| candidate.as_ref().trim().to_string()) + .filter(|candidate| !candidate.is_empty()) + .map(|candidate| self.normalize(&candidate).intern()) + .collect() + } + + pub fn match_epg_channel_candidates(&mut self, epg_id: &Arc, normalized_candidates: &[Arc]) -> bool { + let mut matched = self.match_with_normalized(epg_id, normalized_candidates); + if !matched && self.fuzzy_match_enabled { + if let Some(key) = self.find_best_fuzzy_match_for_candidates(normalized_candidates) { + self.normalized.entry(key).and_modify(|entry| { + entry.replace(epg_id.clone()); + with_folded_epg_id(epg_id, |folded| self.channel_epg_id.insert(folded.intern())); + matched = true; + }); + } + } + matched + } + + fn find_best_fuzzy_match_for_candidates(&self, normalized_candidates: &[Arc]) -> Option> { + let match_threshold = self.smart_match_config.match_threshold; + let best_match_threshold = self.smart_match_config.best_match_threshold; + let pre = normalized_candidates + .iter() + .map(|candidate| (candidate.clone(), self.phonetic(candidate))) + .collect::>(); + + for (candidate, code) in &pre { + if let Some(keys) = self.phonetics.get(code) { + if let Some(good_enough) = keys.iter().find(|norm_key| { + let jw = strsim::jaro_winkler(norm_key, candidate); + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let score = ((jw * 100.0).round() as u16).min(100); + score >= best_match_threshold + }) { + return Some(good_enough.clone()); + } + } + } + + pre.iter() + .filter_map(|(candidate, code)| { + self.phonetics.get(code).and_then(|keys| { + keys.iter() + .map(|norm_key| { + let jw = strsim::jaro_winkler(norm_key, candidate); + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let score = ((jw * 100.0).round() as u16).min(100); + (score, norm_key) + }) + .max_by_key(|(score, _)| *score) + }) + }) + .max_by_key(|(score, _)| *score) + .and_then(|(score, key)| (score >= match_threshold).then(|| key.clone())) + } } /// Assigns EPG IDs and logos to live playlist channels by matching them with EPG data. @@ -319,14 +384,17 @@ pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec>) -> TVGuide { + TVGuide::new(vec![PersistedEpgSource { + file_path: path, + priority: 0, + logo_override: false, + kind: PersistedEpgSourceKind::Ics { + channel_id: "f1.calendar".intern(), + channel_title: Some("Formula 1".intern()), + match_names, + config: Box::new(IcsEpgSourceConfig::default()), + }, + }]) + } + + fn live_playlist_item(name: &str, epg_channel_id: Option<&str>) -> shared::model::PlaylistItem { + shared::model::PlaylistItem { + header: PlaylistItemHeader { + name: name.intern(), + epg_channel_id: epg_channel_id.map(Internable::intern), + xtream_cluster: super::XtreamCluster::Live, + item_type: PlaylistItemType::Live, + ..PlaylistItemHeader::default() + }, + } + } + #[test] fn channel_epg_id_membership_is_case_insensitive_ascii() { let mut cache = super::EpgIdCache::new(None); @@ -399,6 +501,7 @@ mod tests { file_path: epg_path, priority: 0, logo_override: true, + kind: PersistedEpgSourceKind::Xmltv, }]); let mut playlist = FetchedPlaylist { input: &input, @@ -417,6 +520,79 @@ mod tests { }); } + #[test] + fn playlist_item_with_ics_epg_channel_id_gets_ics_programmes() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let dir = tempdir().unwrap(); + let ics_path = dir.path().join("f1.ics"); + write_ics_file(&ics_path); + + let mut input = ConfigInput::from(ConfigInputDto::default()); + input.epg = Some(EpgConfig { sources: vec![], smart_match: None }); + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![live_playlist_item("Formula 1", Some("f1.calendar"))], + xtream_cluster: super::XtreamCluster::Live, + }]; + let mut playlist = FetchedPlaylist { + input: &input, + source: MemoryPlaylistSource::new(groups).into_source(), + epg: Some(ics_tv_guide(ics_path, Vec::new())), + }; + let mut epg = Vec::new(); + + super::process_playlist_epg(&mut playlist, &mut epg).await; + + assert_eq!(epg.len(), 1); + assert_eq!(epg[0].children[0].id.as_ref(), "f1.calendar"); + assert_eq!(epg[0].children[0].programmes.len(), 1); + assert_eq!(epg[0].children[0].programmes[0].title.as_deref(), Some("Practice 1")); + }); + } + + #[test] + fn smart_match_uses_ics_channel_title_and_match_names() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let dir = tempdir().unwrap(); + let ics_path = dir.path().join("f1.ics"); + write_ics_file(&ics_path); + + let mut smart_dto = shared::model::EpgSmartMatchConfigDto { + enabled: true, + fuzzy_matching: false, + ..shared::model::EpgSmartMatchConfigDto::default() + }; + smart_dto.prepare().expect("smart config"); + let mut input = ConfigInput::from(ConfigInputDto::default()); + input.epg = Some(EpgConfig { + sources: vec![], + smart_match: Some(EpgSmartMatchConfig::from(smart_dto)), + }); + let groups = vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![live_playlist_item("F1", None)], + xtream_cluster: super::XtreamCluster::Live, + }]; + let mut playlist = FetchedPlaylist { + input: &input, + source: MemoryPlaylistSource::new(groups).into_source(), + epg: Some(ics_tv_guide(ics_path, vec!["F1".intern()])), + }; + let mut epg = Vec::new(); + + super::process_playlist_epg(&mut playlist, &mut epg).await; + + let updated = playlist.items_mut().next().unwrap(); + assert_eq!(updated.header.epg_channel_id.as_deref(), Some("f1.calendar")); + assert_eq!(epg[0].children[0].id.as_ref(), "f1.calendar"); + assert_eq!(epg[0].children[0].programmes.len(), 1); + }); + } + #[test] fn test_phonetic() { let strings: Vec = (0..5_000) diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index 252ed0003..61ebaf7e8 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -158,12 +158,47 @@ pub async fn epg_query_channels( #[cfg(test)] mod tests { use super::*; + use crate::model::{IcsEpgSourceConfig, M3uTargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput}; + use crate::processing::parser::ics::parse_ics_file_to_channel; use crate::repository::BPlusTree; use crate::utils::FileLockManager; - use shared::model::EpgChannel; + use arc_swap::ArcSwapOption; + use shared::foundation::Filter; + use shared::model::{EpgChannel, ProcessingOrder}; use shared::utils::Internable; use tempfile::TempDir; + fn target_with_m3u_and_xtream() -> ConfigTarget { + ConfigTarget { + id: 1, + enabled: true, + name: "ics-target".to_string(), + options: None, + sort: None, + filter: Filter::default(), + output: vec![ + TargetOutput::M3u(M3uTargetOutput { + filename: None, + include_type_in_url: false, + mask_redirect_url: false, + filter: None, + }), + TargetOutput::Xtream(XtreamTargetOutput { + flags: XtreamTargetFlagsSet::new(), + trakt: None, + filter: None, + }), + ], + rename: None, + mapping_ids: None, + mapping: Arc::new(ArcSwapOption::new(None)), + favourites: None, + processing_order: ProcessingOrder::default(), + watch: None, + use_memory_cache: false, + } + } + #[tokio::test] async fn epg_query_channels_returns_found_channels_in_order() { let tmp = TempDir::new().expect("temp dir created"); @@ -199,4 +234,71 @@ mod tests { assert_eq!(results[2].0.as_ref(), "ch3"); assert!(results[2].1.is_none()); } + + #[tokio::test] + async fn imported_ics_epg_uses_shared_m3u_and_xtream_target_write_read_path() { + let tmp = TempDir::new().expect("temp dir created"); + let ics_path = tmp.path().join("calendar.ics"); + std::fs::write( + &ics_path, + concat!( + "BEGIN:VCALENDAR\r\n", + "VERSION:2.0\r\n", + "BEGIN:VEVENT\r\n", + "UID:f1-session\r\n", + "DTSTART:20300101T120000Z\r\n", + "DTEND:20300101T130000Z\r\n", + "SUMMARY:Formula 1 Practice\r\n", + "DESCRIPTION:Imported from ICS\r\n", + "END:VEVENT\r\n", + "END:VCALENDAR\r\n", + ), + ) + .expect("write ICS fixture"); + + let channel = parse_ics_file_to_channel( + &ics_path, + "f1.calendar".intern(), + Some("Formula 1".intern()), + &IcsEpgSourceConfig::default(), + ) + .await + .expect("parse ICS fixture"); + let epg = Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(channel)], + }; + + let config = Config { + storage_dir: tmp.path().to_string_lossy().into_owned(), + ..Config::default() + }; + let target = target_with_m3u_and_xtream(); + let target_path = crate::repository::get_target_storage_path(&config, &target.name) + .expect("target storage path"); + let m3u_path = m3u_get_epg_file_path_for_target(&target_path); + let xtream_storage = xtream_get_storage_path(&config, &target.name).expect("xtream storage path"); + let xtream_path = xtream_get_epg_file_path_for_target(&xtream_storage); + std::fs::create_dir_all(m3u_path.parent().expect("m3u parent")).expect("create m3u storage"); + std::fs::create_dir_all(xtream_path.parent().expect("xtream parent")).expect("create xtream storage"); + + for output in &target.output { + epg_write_for_target(&config, &target, &target_path, Some(&epg), output, None) + .await + .expect("write target EPG"); + } + + let locks = FileLockManager::new(); + for epg_path in [&m3u_path, &xtream_path] { + let results = epg_query_channels(&locks, epg_path, vec!["f1.calendar".intern()]) + .await + .expect("read target EPG"); + let stored = results[0].1.as_ref().expect("stored ICS channel"); + assert_eq!(stored.title.as_deref(), Some("Formula 1")); + assert_eq!(stored.programmes.len(), 1); + assert_eq!(stored.programmes[0].title.as_deref(), Some("Formula 1 Practice")); + } + } } diff --git a/backend/src/utils/network/epg.rs b/backend/src/utils/network/epg.rs index efd61363c..0a8b8e7cd 100644 --- a/backend/src/utils/network/epg.rs +++ b/backend/src/utils/network/epg.rs @@ -1,40 +1,94 @@ -use crate::model::{ConfigInput, PersistedEpgSource}; -use crate::model::{TVGuide}; +use crate::model::{ + ConfigInput, EpgSource, EpgSourceType, PersistedEpgSource, PersistedEpgSourceKind, TVGuide, +}; use crate::processing::processor::PlaylistProcessingContext; use crate::repository::get_input_storage_path; -use crate::repository::storage_const; use crate::utils::{add_prefix_to_filename, prepare_file_path, request}; -use crate::utils::cleanup_unlisted_files_with_suffix; use log::debug; use shared::concat_string; -use shared::error::{ TuliproxError}; +use shared::error::TuliproxError; use shared::utils::{sanitize_sensitive_info, short_hash}; -use std::path::PathBuf; +use std::{collections::HashSet, path::{Path, PathBuf}}; -pub async fn get_input_raw_epg_file_path(url: &str, input: &ConfigInput, storage_dir: &str) -> std::io::Result { - let file_prefix = short_hash(url); +pub async fn get_input_raw_epg_file_path( + epg_source: &EpgSource, + input: &ConfigInput, + storage_dir: &str, +) -> std::io::Result { + let cache_key = epg_cache_key(epg_source); + let file_prefix = short_hash(&cache_key); + let suffix = epg_source_cache_suffix(epg_source.source_type); + let extension = epg_source_cache_extension(epg_source.source_type); if let Some(persist_path) = input.persist.as_deref() { if !persist_path.is_empty() { - if let Some(path) = prepare_file_path(input.persist.as_deref(), storage_dir, "") - .map(|path| add_prefix_to_filename(&path, concat_string!(&file_prefix, "_epg_").as_str(), Some("xml"))) { + if let Some(path) = prepare_file_path(input.persist.as_deref(), storage_dir, "").map(|path| { + add_prefix_to_filename(&path, concat_string!(&file_prefix, "_epg_").as_str(), Some(extension)) + }) { return Ok(path); } } } let download_path = get_input_storage_path(&input.name, storage_dir).await?; - Ok(download_path.join(format!("{}_{}", file_prefix, storage_const::FILE_EPG))) + Ok(download_path.join(format!("{file_prefix}{suffix}"))) } -async fn download_epg_file(url: &str, ctx: &PlaylistProcessingContext, - input: &ConfigInput, - headers: Option<&reqwest::header::HeaderMap>, - storage_dir: &str) -> Result { - debug!("Getting epg file path for url: {}", sanitize_sensitive_info(url)); - let persist_file_path = get_input_raw_epg_file_path(url, input, storage_dir).await.map_err(|e| TuliproxError::Io(format!("Could not access epg file download directory: {e}")))?; +pub async fn get_input_raw_xmltv_file_path( + url: &str, + input: &ConfigInput, + storage_dir: &str, +) -> std::io::Result { + let source = EpgSource { + source_type: EpgSourceType::Xmltv, + url: url.to_string(), + priority: 0, + logo_override: false, + channel_id: None, + channel_title: None, + match_names: Vec::new(), + ics: None, + }; + get_input_raw_epg_file_path(&source, input, storage_dir).await +} + +fn epg_cache_key(epg_source: &EpgSource) -> String { + match epg_source.source_type { + EpgSourceType::Xmltv => epg_source.url.clone(), + EpgSourceType::Ics => { + format!("ics|{}|{}", epg_source.url, epg_source.channel_id.as_deref().unwrap_or_default()) + } + } +} + +fn epg_source_cache_suffix(source_type: EpgSourceType) -> &'static str { + match source_type { + EpgSourceType::Xmltv => "_epg.xml", + EpgSourceType::Ics => "_epg.ics", + } +} + +fn epg_source_cache_extension(source_type: EpgSourceType) -> &'static str { + match source_type { + EpgSourceType::Xmltv => "xml", + EpgSourceType::Ics => "ics", + } +} + +async fn download_epg_file( + epg_source: &EpgSource, + ctx: &PlaylistProcessingContext, + input: &ConfigInput, + headers: Option<&reqwest::header::HeaderMap>, + storage_dir: &str, +) -> Result { + debug!("Getting epg file path for url: {}", sanitize_sensitive_info(&epg_source.url)); + let persist_file_path = get_input_raw_epg_file_path(epg_source, input, storage_dir) + .await + .map_err(|e| TuliproxError::Io(format!("Could not access epg file download directory: {e}")))?; if input.cache_duration_seconds > 0 { + let _cache_lock = ctx.config.file_locks.read_lock(&persist_file_path).await; if let Ok(metadata) = tokio::fs::metadata(&persist_file_path).await { if let Ok(modified) = metadata.modified() { if let Ok(elapsed) = std::time::SystemTime::now().duration_since(modified) { @@ -45,28 +99,87 @@ async fn download_epg_file(url: &str, ctx: &PlaylistProcessingContext, } } } - // Cache miss: file doesn't exist } let lock_key: std::sync::Arc = persist_file_path.display().to_string().into(); let _input_lock = ctx.get_input_lock(&lock_key).await; if ctx.is_input_downloaded(&lock_key).await { - return Ok(persist_file_path); + let _cache_lock = ctx.config.file_locks.read_lock(&persist_file_path).await; + if tokio::fs::metadata(&persist_file_path).await.is_ok_and(|metadata| metadata.is_file()) { + return Ok(persist_file_path); + } } + debug!("Downloading epg for input '{}'", input.name); - match request::get_input_epg_content_as_file(&ctx.config, &ctx.client, input, headers, storage_dir, url, &persist_file_path).await { + let max_bytes = epg_download_limit(epg_source); + match request::get_input_epg_content_as_file( + &ctx.config, + &ctx.client, + input, + request::InputEpgFileRequest { + headers, + storage_dir, + url: &epg_source.url, + persist_path: &persist_file_path, + max_bytes, + }, + ) + .await + { Ok(path) => { ctx.mark_input_downloaded(lock_key.clone()).await; Ok(path) } - Err(err) => Err(err) + Err(err) => Err(err), } } -pub async fn get_xmltv(ctx: &PlaylistProcessingContext, input: &ConfigInput, - headers: Option<&reqwest::header::HeaderMap>, - storage_dir: &str) -> (Option, Vec) { +fn epg_download_limit(epg_source: &EpgSource) -> Option { + match epg_source.source_type { + EpgSourceType::Xmltv => None, + EpgSourceType::Ics => epg_source.ics.as_ref().map(|config| config.max_download_bytes), + } +} + +async fn cleanup_unlisted_epg_files( + file_locks: &crate::utils::FileLockManager, + keep_files: &[PathBuf], + suffix: &str, +) -> std::io::Result<()> { + let keep_set: HashSet<&Path> = keep_files.iter().map(PathBuf::as_path).collect(); + let directories: HashSet<&Path> = keep_files.iter().filter_map(|file| file.parent()).collect(); + + for directory in directories { + let mut entries = tokio::fs::read_dir(directory).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let suffix_matches = + path.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.ends_with(suffix)); + if keep_set.contains(path.as_path()) || !suffix_matches { + continue; + } + + let _file_lock = file_locks.write_lock(&path).await; + if tokio::fs::metadata(&path).await.is_ok_and(|metadata| metadata.is_file()) { + match tokio::fs::remove_file(&path).await { + Ok(()) => log::trace!("Deleted {:?}", path.display()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + } + } + } + + Ok(()) +} + +pub async fn get_xmltv( + ctx: &PlaylistProcessingContext, + input: &ConfigInput, + headers: Option<&reqwest::header::HeaderMap>, + storage_dir: &str, +) -> (Option, Vec) { match &input.epg { None => (None, vec![]), Some(epg_config) => { @@ -75,10 +188,13 @@ pub async fn get_xmltv(ctx: &PlaylistProcessingContext, input: &ConfigInput, let mut stored_file_paths = vec![]; for epg_source in &epg_config.sources { - match download_epg_file(&epg_source.url, ctx, input, headers, storage_dir).await { + match download_epg_file(epg_source, ctx, input, headers, storage_dir).await { Ok(file_path) => { stored_file_paths.push(file_path.clone()); - file_paths.push(PersistedEpgSource { file_path, priority: epg_source.priority, logo_override: epg_source.logo_override }); + match persisted_source_from_config(epg_source, file_path) { + Ok(persisted) => file_paths.push(persisted), + Err(err) => errors.push(err), + } } Err(err) => { errors.push(err); @@ -86,13 +202,170 @@ pub async fn get_xmltv(ctx: &PlaylistProcessingContext, input: &ConfigInput, } } - let _ = cleanup_unlisted_files_with_suffix(&stored_file_paths, "_epg.xml"); + let _ = cleanup_unlisted_epg_files(&ctx.config.file_locks, &stored_file_paths, "_epg.xml").await; + let _ = cleanup_unlisted_epg_files(&ctx.config.file_locks, &stored_file_paths, "_epg.ics").await; if file_paths.is_empty() { (None, errors) } else { - (Some(TVGuide::new(file_paths)), errors) + ( + Some(TVGuide::new(file_paths).with_file_locks(std::sync::Arc::clone(&ctx.config.file_locks))), + errors, + ) } } } } + +fn persisted_source_from_config( + epg_source: &EpgSource, + file_path: PathBuf, +) -> Result { + let kind = match epg_source.source_type { + EpgSourceType::Xmltv => PersistedEpgSourceKind::Xmltv, + EpgSourceType::Ics => { + let channel_id = epg_source.channel_id.clone().ok_or_else(|| { + TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string()) + })?; + PersistedEpgSourceKind::Ics { + channel_id, + channel_title: epg_source.channel_title.clone(), + match_names: epg_source.match_names.clone(), + config: Box::new(epg_source.ics.clone().unwrap_or_default()), + } + } + }; + + Ok(PersistedEpgSource { + file_path, + priority: epg_source.priority, + logo_override: epg_source.logo_override, + kind, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{model::ConfigInput, utils::FileLockManager}; + use shared::utils::Internable; + use tempfile::tempdir; + + fn source(source_type: EpgSourceType, url: &str, channel_id: Option<&str>) -> EpgSource { + EpgSource { + source_type, + url: url.to_string(), + priority: 0, + logo_override: false, + channel_id: channel_id.map(Internable::intern), + channel_title: None, + match_names: Vec::new(), + ics: None, + } + } + + #[tokio::test] + async fn epg_cache_path_uses_xmltv_suffix() { + let dir = tempdir().expect("temp dir"); + let input = ConfigInput { + name: "input".intern(), + ..ConfigInput::default() + }; + let path = get_input_raw_epg_file_path( + &source(EpgSourceType::Xmltv, "https://example.com/xmltv.xml", None), + &input, + dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("path"); + assert!(path.to_string_lossy().ends_with("_epg.xml")); + } + + #[tokio::test] + async fn epg_cache_path_uses_ics_suffix() { + let dir = tempdir().expect("temp dir"); + let input = ConfigInput { + name: "input".intern(), + ..ConfigInput::default() + }; + let path = get_input_raw_epg_file_path( + &source(EpgSourceType::Ics, "https://example.com/f1.ics", Some("f1.calendar")), + &input, + dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("path"); + assert!(path.to_string_lossy().ends_with("_epg.ics")); + } + + #[tokio::test] + async fn same_ics_url_with_different_channel_id_gets_different_cache_path() { + let dir = tempdir().expect("temp dir"); + let input = ConfigInput { + name: "input".intern(), + ..ConfigInput::default() + }; + let first = get_input_raw_epg_file_path( + &source(EpgSourceType::Ics, "https://example.com/calendar.ics", Some("one")), + &input, + dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("first path"); + let second = get_input_raw_epg_file_path( + &source(EpgSourceType::Ics, "https://example.com/calendar.ics", Some("two")), + &input, + dir.path().to_string_lossy().as_ref(), + ) + .await + .expect("second path"); + assert_ne!(first, second); + } + + #[tokio::test] + async fn epg_cleanup_waits_for_readers_and_only_removes_unlisted_matching_files() { + let dir = tempdir().expect("temp dir"); + let keep = dir.path().join("keep_epg.ics"); + let stale = dir.path().join("stale_epg.ics"); + let other = dir.path().join("other_epg.xml"); + tokio::fs::write(&keep, b"keep").await.expect("write kept file"); + tokio::fs::write(&stale, b"stale").await.expect("write stale file"); + tokio::fs::write(&other, b"other").await.expect("write non-matching file"); + let file_locks = FileLockManager::default(); + let read_guard = file_locks.read_lock(&stale).await; + let keep_files = vec![keep.clone()]; + let cleanup = cleanup_unlisted_epg_files(&file_locks, &keep_files, "_epg.ics"); + tokio::pin!(cleanup); + + assert!(tokio::time::timeout(std::time::Duration::from_millis(25), cleanup.as_mut()).await.is_err()); + assert!(stale.exists()); + + drop(read_guard); + cleanup.await.expect("cleanup after reader release"); + assert!(keep.exists()); + assert!(!stale.exists()); + assert!(other.exists()); + } + + #[test] + fn xmltv_source_does_not_apply_ics_download_limit() { + let mut epg_source = source(EpgSourceType::Xmltv, "https://example.com/xmltv.xml", None); + epg_source.ics = Some(crate::model::IcsEpgSourceConfig { + max_download_bytes: 123, + ..crate::model::IcsEpgSourceConfig::default() + }); + + assert_eq!(epg_download_limit(&epg_source), None); + } + + #[test] + fn ics_source_applies_configured_download_limit() { + let mut epg_source = source(EpgSourceType::Ics, "https://example.com/calendar.ics", Some("calendar")); + epg_source.ics = Some(crate::model::IcsEpgSourceConfig { + max_download_bytes: 123, + ..crate::model::IcsEpgSourceConfig::default() + }); + + assert_eq!(epg_download_limit(&epg_source), Some(123)); + } +} diff --git a/backend/src/utils/network/request.rs b/backend/src/utils/network/request.rs index 75949e5b7..3930bfcc2 100644 --- a/backend/src/utils/network/request.rs +++ b/backend/src/utils/network/request.rs @@ -61,6 +61,20 @@ impl RequestFetchOptions { } } +#[derive(Debug, Clone, Copy, Default)] +pub struct FileDownloadOptions { + pub max_bytes: Option, + pub atomic_write: bool, +} + +pub struct InputEpgFileRequest<'a> { + pub headers: Option<&'a HeaderMap>, + pub storage_dir: &'a str, + pub url: &'a str, + pub persist_path: &'a Path, + pub max_bytes: Option, +} + fn log_proxy_diagnostics(config: &Config) { PROXY_DIAGNOSTICS_ONCE.call_once(|| { if let Some(proxy_cfg) = config.proxy.as_ref() { @@ -1361,18 +1375,29 @@ pub async fn get_input_epg_content_as_file( app_config: &Arc, client: &reqwest::Client, input: &ConfigInput, - headers: Option<&HeaderMap>, - storage_dir: &str, - url_str: &str, - persist_filepath: &Path, + request: InputEpgFileRequest<'_>, ) -> Result { + let InputEpgFileRequest { + headers, + storage_dir, + url: url_str, + persist_path, + max_bytes, + } = request; debug_if_enabled!( "getting input epg content storage_dir: {}, url: {}", storage_dir, sanitize_sensitive_info(url_str) ); - if url_str.parse::().is_ok() { - match download_epg_content_as_file(app_config, client, input, headers, url_str, persist_filepath).await { + + // This is the single write-lock boundary for EPG cache population. Callers must + // not hold a lock for `persist_path` while invoking this function. + let _persist_lock = app_config.file_locks.write_lock(persist_path).await; + + // On Windows, drive-letter paths also parse as URLs (with `c` as the scheme). + // Interpret an absolute platform path before attempting URL parsing. + if !Path::new(url_str).is_absolute() && url_str.parse::().is_ok() { + match download_epg_content_as_file(app_config, client, input, headers, url_str, persist_path, max_bytes).await { Ok(content) => Ok(content), Err(e) => { error!( @@ -1390,36 +1415,21 @@ pub async fn get_input_epg_content_as_file( } } } else { - let result = match get_file_path(storage_dir, Some(PathBuf::from(url_str))) { - Some(filepath) => { - if filepath.exists() { - if let Err(e) = tokio::fs::copy(&filepath, persist_filepath).await { - error!("can't persist to: {} => {}", persist_filepath.display(), e); - return Err(TuliproxError::RepositoryNetwork(format!("Failed to persist: {} => {}", persist_filepath.display(), e))); - } - if filepath.exists() { - Some(filepath) - } else { - return Err(TuliproxError::RepositoryNetwork(format!( - "Failed: file does not exists {}", - filepath.display() - ))); - } - } else { - None - } - } - None => None, + let Some(file_path) = get_file_path(storage_dir, Some(PathBuf::from(url_str))) else { + let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); + error!("{msg}"); + return Err(TuliproxError::RepositoryNetwork(msg)); }; + if !file_path.exists() { + let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); + error!("{msg}"); + return Err(TuliproxError::RepositoryNetwork(msg)); + } - result.map_or_else( - || { - let msg = format!("can't read input url: {}", sanitize_sensitive_info(url_str)); - error!("{msg}"); - Err(TuliproxError::RepositoryNetwork(msg)) - }, - Ok, - ) + copy_local_epg_file_to_persist(&file_path, persist_path, max_bytes).await.map_err(|err| { + error!("can't persist to: {} => {}", persist_path.display(), err); + TuliproxError::RepositoryNetwork(format!("Failed to persist: {} => {err}", persist_path.display())) + }) } } @@ -1721,32 +1731,60 @@ pub async fn get_remote_content_as_file( url: &Url, file_path: &Path, ) -> Result { - let custom_headers = headers - .map(|h| h.iter().map(|(k, v)| (k.as_str().to_string(), v.as_bytes().to_vec())).collect::>()); + get_remote_content_as_file_with_options( + app_config, + client, + input, + headers, + url, + file_path, + FileDownloadOptions::default(), + ) + .await +} - let config = app_config.config.load(); - let default_user_agent = config.default_user_agent.clone(); - drop(config); +pub async fn get_remote_content_as_file_with_options( + app_config: &Arc, + client: &reqwest::Client, + input: &ConfigInput, + headers: Option<&HeaderMap>, + url: &Url, + file_path: &Path, + options: FileDownloadOptions, +) -> Result { + let input_source = InputSource { + name: input.name.clone(), + url: url.to_string(), + provider: input.get_resolve_provider(url.as_str()), + username: input.username.clone(), + password: input.password.clone(), + method: input.method, + headers: input.headers.clone(), + }; - let provider_config = input.get_resolve_provider(url.as_str()); - - let response = send_with_retry_and_provider(app_config, url, provider_config.as_ref(), false, |resolved_url| { - get_client_request( - client, - input.method, - Some(&input.headers), - resolved_url, - custom_headers.as_ref(), - None, - default_user_agent.as_deref(), - ) - }) - .await?; + let response = send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result( + app_config, + client, + &input_source, + headers, + url, + 10, + RequestFetchOptions::default(), + ) + .await? + .response; let start_time = tokio::time::Instant::now(); - let mut writer = async_file_writer(File::create(file_path).await?); + let (temp_file, output_file) = if options.atomic_write { + let (temp_file, output_file) = create_atomic_download_file(file_path)?; + (Some(temp_file), output_file) + } else { + (None, File::create(file_path).await?) + }; + let mut writer = async_file_writer(output_file); let mut stream = response.bytes_stream(); + let mut downloaded = 0_u64; let idle_timeout = tokio::time::Duration::from_secs(STREAM_IDLE_TIMEOUT); let idle = sleep(idle_timeout); @@ -1756,7 +1794,10 @@ pub async fn get_remote_content_as_file( tokio::select! { () = &mut idle => { warn!("Stream idle for request, closing {}", sanitize_sensitive_info(url.as_ref())); - break; + return Err(string_to_io_error(format!( + "Download timed out for {}", + sanitize_sensitive_info(url.as_ref()) + ))); } chunk = stream.next() => { @@ -1764,6 +1805,18 @@ pub async fn get_remote_content_as_file( match chunk { Some(Ok(bytes)) => { + downloaded = downloaded.checked_add(bytes.len() as u64).ok_or_else(|| { + string_to_io_error(format!( + "Download size overflow for {}", + sanitize_sensitive_info(url.as_ref()) + )) + })?; + if options.max_bytes.is_some_and(|max| downloaded > max) { + return Err(string_to_io_error(format!( + "Download exceeds configured limit for {}", + sanitize_sensitive_info(url.as_ref()) + ))); + } writer.write_all(&bytes).await?; } Some(Err(e)) => { @@ -1779,6 +1832,11 @@ pub async fn get_remote_content_as_file( writer.flush().await?; writer.shutdown().await?; + drop(writer); + + if let Some(temp_file) = temp_file { + persist_atomic_download_file(temp_file, file_path)?; + } debug!( "File downloaded successfully to {}, took {}", @@ -2130,6 +2188,62 @@ fn strip_sensitive_headers_for_cross_origin_redirect(headers: &mut HashMap Result<(tempfile::NamedTempFile, File), Error> { + let parent = file_path + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let temp_file = tempfile::Builder::new() + .prefix(".tuliprox-download-") + .suffix(".tmp") + .tempfile_in(parent)?; + let output_file = File::from_std(temp_file.reopen()?); + Ok((temp_file, output_file)) +} + +fn persist_atomic_download_file(temp_file: tempfile::NamedTempFile, file_path: &Path) -> Result<(), Error> { + match temp_file.persist(file_path) { + Ok(_) => Ok(()), + Err(err) => Err(err.error), + } +} + +async fn copy_local_epg_file_to_persist( + file_path: &Path, + persist_filepath: &Path, + max_bytes: Option, +) -> Result { + let mut reader = File::open(file_path).await?; + let (temp_file, output_file) = create_atomic_download_file(persist_filepath)?; + let mut writer = async_file_writer(output_file); + let mut copied = 0_u64; + let mut buffer = vec![0_u8; 64 * 1024].into_boxed_slice(); + + loop { + let read = reader.read(&mut buffer).await?; + if read == 0 { + break; + } + copied = copied.checked_add(read as u64).ok_or_else(|| { + string_to_io_error(format!("Local EPG file size overflow for {}", file_path.display())) + })?; + if max_bytes.is_some_and(|max| copied > max) { + return Err(string_to_io_error(format!( + "Local EPG file {} exceeds configured limit", + file_path.display() + ))); + } + writer.write_all(&buffer[..read]).await?; + } + + writer.flush().await?; + writer.shutdown().await?; + drop(writer); + drop(reader); + persist_atomic_download_file(temp_file, persist_filepath)?; + Ok(persist_filepath.to_path_buf()) +} + async fn download_epg_content_as_file( app_config: &Arc, client: &reqwest::Client, @@ -2137,26 +2251,42 @@ async fn download_epg_content_as_file( headers: Option<&HeaderMap>, url_str: &str, persist_filepath: &Path, + max_bytes: Option, ) -> Result { if let Ok(url) = url_str.parse::() { - if url.scheme() == "file" { - url.to_file_path().map_or_else( - |()| { - Err(Error::new( + match url.scheme() { + "file" => { + let file_path = url.to_file_path().map_err(|()| { + Error::new( ErrorKind::Unsupported, format!("Unknown file {}", sanitize_sensitive_info(url_str)), - )) - }, - |file_path| { - if file_path.exists() { - Ok(file_path) - } else { - Err(Error::new(ErrorKind::NotFound, format!("Unknown file {}", file_path.display()))) - } - }, - ) - } else { - get_remote_content_as_file(app_config, client, input, headers, &url, persist_filepath).await + ) + })?; + if file_path.exists() { + copy_local_epg_file_to_persist(&file_path, persist_filepath, max_bytes).await + } else { + Err(Error::new(ErrorKind::NotFound, format!("Unknown file {}", file_path.display()))) + } + } + "http" | "https" | "provider" => { + get_remote_content_as_file_with_options( + app_config, + client, + input, + headers, + &url, + persist_filepath, + FileDownloadOptions { + max_bytes, + atomic_write: true, + }, + ) + .await + } + scheme => Err(Error::new( + ErrorKind::Unsupported, + format!("Unsupported EPG URL scheme '{scheme}' for {}", sanitize_sensitive_info(url_str)), + )), } } else { Err(Error::new(ErrorKind::Unsupported, format!("Malformed URL {}", sanitize_sensitive_info(url_str)))) @@ -2608,14 +2738,14 @@ pub fn should_trigger_failover(status: StatusCode) -> bool { mod tests { use super::{ is_safe_cross_origin_redirect_header, - next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, + get_input_epg_content_as_file, next_provider_url_index, preview_request_diagnostics_for_logging, preview_request_target_for_logging, resolve_attempt_target, same_origin, send_input_with_retry_and_provider_policy_with_manual_redirects_and_options_result, send_with_retry_and_provider, send_with_retry_and_provider_policy, should_try_next_ip_on_connect_error, - strip_sensitive_headers_for_cross_origin_redirect, RequestFetchOptions, + strip_sensitive_headers_for_cross_origin_redirect, InputEpgFileRequest, RequestFetchOptions, }; use crate::{ model::{ - AppConfig, Config, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, + AppConfig, Config, ConfigInput, ConfigProvider, InputSource, MediaToolCapabilities, ResourceRetryConfig, ReverseProxyConfig, SourcesConfig, }, utils::{FileLockManager}, @@ -2631,6 +2761,7 @@ mod tests { use std::{ collections::{HashMap, HashSet}, net::SocketAddr, + path::{Path, PathBuf}, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -2670,6 +2801,44 @@ mod tests { }) } + fn make_epg_test_client() -> reqwest::Client { + reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(2)) + .build() + .expect("test client") + } + + async fn atomic_download_temp_files(directory: &Path) -> Vec { + let mut result = Vec::new(); + let mut entries = tokio::fs::read_dir(directory).await.expect("read temp directory"); + while let Some(entry) = entries.next_entry().await.expect("read temp directory entry") { + if entry.file_name().to_string_lossy().starts_with(".tuliprox-download-") { + result.push(entry.path()); + } + } + result + } + + #[tokio::test] + async fn atomic_download_temp_files_are_unique_and_use_target_directory() { + let dir = tempfile::tempdir().expect("temp dir"); + let target = dir.path().join("cache.ics"); + let (first_temp, first_output) = super::create_atomic_download_file(&target).expect("first temp file"); + let (second_temp, second_output) = super::create_atomic_download_file(&target).expect("second temp file"); + + assert_ne!(first_temp.path(), second_temp.path()); + assert_eq!(first_temp.path().parent(), Some(dir.path())); + assert_eq!(second_temp.path().parent(), Some(dir.path())); + + drop(first_output); + drop(second_output); + drop(first_temp); + drop(second_temp); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + fn make_provider_with_dns(keep_vhost: bool, on_connect_error: OnConnectErrorPolicy, ips: Vec<&str>) -> Arc { let parsed_ips = ips .into_iter() @@ -2802,6 +2971,367 @@ mod tests { assert!(!is_safe_cross_origin_redirect_header("x-auth-token")); } + #[tokio::test] + async fn local_epg_file_respects_max_download_bytes() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("large.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"0123456789").await.expect("write source"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source.to_string_lossy().as_ref(), + persist_path: &persist, + max_bytes: Some(4), + }, + ) + .await + .expect_err("size limit should fail"); + + assert!(err.to_string().contains("exceeds configured limit")); + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + + #[tokio::test] + async fn file_url_epg_source_is_copied_to_persist_path() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n") + .await + .expect("write source"); + tokio::fs::write(&persist, b"old cache").await.expect("write old cache"); + let source_url = Url::from_file_path(&source).expect("file url"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let result = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source_url.as_str(), + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect("download"); + + assert_eq!(result, persist); + assert_eq!( + tokio::fs::read_to_string(&persist).await.expect("persisted content"), + "BEGIN:VCALENDAR\nEND:VCALENDAR\n" + ); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + + #[tokio::test] + async fn file_url_epg_source_respects_streamed_size_limit() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("large.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"0123456789").await.expect("write source"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let source_url = Url::from_file_path(&source).expect("file url"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source_url.as_str(), + persist_path: &persist, + max_bytes: Some(4), + }, + ) + .await + .expect_err("size limit should fail"); + + assert!(err.to_string().contains("exceeds configured limit")); + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + + #[tokio::test] + async fn local_epg_replace_error_cleans_temp_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n") + .await + .expect("write source"); + tokio::fs::create_dir(&persist).await.expect("create conflicting destination"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source.to_string_lossy().as_ref(), + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("replacing a directory should fail"); + + assert!(persist.is_dir()); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + + #[tokio::test] + async fn local_epg_read_error_preserves_existing_cache_and_cleans_temp_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source-directory"); + let persist = dir.path().join("cache.ics"); + tokio::fs::create_dir(&source).await.expect("create source directory"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source.to_string_lossy().as_ref(), + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("reading a directory as an EPG file should fail"); + + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + + #[tokio::test] + async fn unsupported_epg_url_scheme_is_rejected_without_network_access() { + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: "ftp://example.com/calendar.ics", + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("unsupported scheme should fail"); + + assert!(err.to_string().contains("Unsupported EPG URL scheme 'ftp'")); + assert!(!persist.exists()); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + } + + #[cfg(windows)] + #[tokio::test] + async fn absolute_windows_epg_path_is_dispatched_as_local_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let source = dir.path().join("source.ics"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&source, b"BEGIN:VCALENDAR\nEND:VCALENDAR\n") + .await + .expect("write source"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + let source_path = source.to_str().expect("Windows temp path should be UTF-8"); + assert!(source_path.contains(':')); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: source_path, + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect("absolute Windows path should be copied as a local file"); + + assert_eq!(tokio::fs::read(&persist).await.expect("read cache"), b"BEGIN:VCALENDAR\nEND:VCALENDAR\n"); + } + + #[tokio::test] + async fn remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file() { + let (addr, _accepted, server_handle) = match start_plain_http_server_with_body(b"0123456789").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping remote_epg_size_limit_preserves_existing_cache_and_cleans_temp_file: {err}"); + return; + } + Err(err) => panic!("failed to start test server: {err}"), + }; + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let url = format!("http://{addr}/calendar.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + let err = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(4), + }, + ) + .await + .expect_err("remote size limit should fail"); + + assert!(err.to_string().contains("exceeds configured limit")); + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + server_handle.abort(); + } + + #[tokio::test] + async fn remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file() { + let response = "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\ntruncated".to_string(); + let (addr, _accepted, server_handle) = match start_plain_http_server_with_response(response).await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!("skipping remote_epg_stream_error_preserves_existing_cache_and_cleans_temp_file: {err}"); + return; + } + Err(err) => panic!("failed to start test server: {err}"), + }; + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let url = format!("http://{addr}/calendar.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + + get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: dir.path().to_string_lossy().as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(1024), + }, + ) + .await + .expect_err("truncated response body should fail"); + + assert_eq!(tokio::fs::read(&persist).await.expect("read existing cache"), b"existing cache"); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + server_handle.abort(); + } + + #[tokio::test] + async fn parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomically() { + let (addr, accepted, max_active, server_handle) = + match start_delayed_http_server_with_body(b"BEGIN:VCALENDAR\nEND:VCALENDAR\n").await { + Ok(server) => server, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + eprintln!( + "skipping parallel_epg_downloads_to_same_target_are_serialized_and_replace_atomically: {err}" + ); + return; + } + Err(err) => panic!("failed to start test server: {err}"), + }; + let dir = tempfile::tempdir().expect("temp dir"); + let persist = dir.path().join("cache.ics"); + tokio::fs::write(&persist, b"existing cache").await.expect("write existing cache"); + let url = format!("http://{addr}/calendar.ics"); + let app_config = make_test_app_config(Config::default()); + let client = make_epg_test_client(); + let input = ConfigInput::default(); + let storage_dir = dir.path().to_string_lossy(); + + let first = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: storage_dir.as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(1024), + }, + ); + let second = get_input_epg_content_as_file( + &app_config, + &client, + &input, + InputEpgFileRequest { + headers: None, + storage_dir: storage_dir.as_ref(), + url: &url, + persist_path: &persist, + max_bytes: Some(1024), + }, + ); + let (first_result, second_result) = tokio::join!(first, second); + + assert_eq!(first_result.expect("first download"), persist); + assert_eq!(second_result.expect("second download"), persist); + assert_eq!(accepted.load(Ordering::SeqCst), 2); + assert_eq!(max_active.load(Ordering::SeqCst), 1); + assert_eq!( + tokio::fs::read(&persist).await.expect("read refreshed cache"), + b"BEGIN:VCALENDAR\nEND:VCALENDAR\n" + ); + assert!(atomic_download_temp_files(dir.path()).await.is_empty()); + server_handle.abort(); + } + #[test] fn test_keep_vhost_false_uses_ip_host_header_for_http() { let provider = make_provider_with_dns(false, OnConnectErrorPolicy::TryNextIp, vec!["192.168.0.1"]); @@ -2966,6 +3496,52 @@ mod tests { Ok((addr, accepted, handle)) } + async fn start_delayed_http_server_with_body( + body: &'static [u8], + ) -> std::io::Result<( + SocketAddr, + Arc, + Arc, + tokio::task::JoinHandle<()>, + )> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let accepted = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let accepted_clone = Arc::clone(&accepted); + let active_clone = Arc::clone(&active); + let max_active_clone = Arc::clone(&max_active); + let content_length = body.len(); + + let handle = tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + continue; + }; + accepted_clone.fetch_add(1, Ordering::SeqCst); + let active = Arc::clone(&active_clone); + let max_active = Arc::clone(&max_active_clone); + tokio::spawn(async move { + let active_count = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(active_count, Ordering::SeqCst); + let mut buf = vec![0_u8; 2048]; + let _ = socket.read(&mut buf).await; + tokio::time::sleep(Duration::from_millis(100)).await; + let response_head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {content_length}\r\nConnection: close\r\n\r\n" + ); + let _ = socket.write_all(response_head.as_bytes()).await; + let _ = socket.write_all(body).await; + let _ = socket.shutdown().await; + active.fetch_sub(1, Ordering::SeqCst); + }); + } + }); + + Ok((addr, accepted, max_active, handle)) + } + async fn start_plain_http_server_with_response( response: String, ) -> std::io::Result<(SocketAddr, Arc, tokio::task::JoinHandle<()>)> { diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index 0eda045e7..3babf8bbc 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -228,50 +228,382 @@ specific provider. ### 2.3 EPG Assignment & Smart Match (`epg`) -Tuliprox can load external XMLTV files and map them intelligently using advanced fuzzy matching to streams that are +Tuliprox can load external EPG sources and map them intelligently using advanced fuzzy matching to streams that are missing a valid EPG ID. -Within the `epg` block, you can define multiple XMLTV providers. -Tuliprox aggregates these sources and assigns EPG data based on priority and matching rules. + +The `epg.sources` list supports the following source types: + +* **XMLTV** sources, which provide complete XMLTV channel and programme data. +* **ICS calendar** sources, which import iCalendar events and convert them into a virtual XMLTV-compatible EPG channel. + +Tuliprox aggregates all configured EPG sources and assigns EPG data based on priority and matching rules. #### Example Configuration ```yaml epg: sources: - - url: "auto" # Automatically generated provider URL - priority: -2 # High priority - logo_override: true # Replaces provider logos with EPG icons + - url: "auto" # Automatically generated provider XMLTV URL + priority: -2 # High priority + logo_override: true # Replaces provider logos with EPG icons + - url: "http://localhost:3001/xmltv.php?epg_id=1" priority: -1 + - url: "http://localhost:3001/xmltv.php?epg_id=2" priority: 3 - - url: "http://localhost:3001/xmltv.php?epg_id=3" - priority: 0 + + - type: ics + url: "https://files-f1.motorsportcalendars.com/f1-calendar_p1_p2_p3_qualifying_sprint_gp.ics" + channel_id: "f1.calendar" + channel_title: "Formula 1" + priority: -10 + ics: + timezone: "Europe/Budapest" + event: + title: "{summary}" + description: "{description}" + include_location: true + include_categories: true + dummy: + enabled: true + title: "No Formula 1 session" + description: "There is currently no scheduled Formula 1 session." + days_past: 1 + days_future: 30 + block_hours: 4 + min_gap_minutes: 1 + smart_match: enabled: true fuzzy_matching: true match_threshold: 80 best_match_threshold: 99 name_prefix: { suffix: "." } - name_prefix_separator: [ ':', '|', '-' ] - strip: [ "3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw" ] + name_prefix_separator: [":", "|", "-"] + strip: ["3840p", "uhd", "fhd", "hd", "sd", "4k", "plus", "raw"] normalize_regex: '[^a-zA-Z0-9\-]' ``` #### EPG Source Parameters (`sources`) -| Parameter | Type | Required | Default | Technical Impact & Background | -|:--------------------|:-------|:--------:|:--------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **`url`** | String | Yes | | The XMLTV endpoint. Use **`auto`** for Xtream inputs to automatically generate the native XMLTV URL using your credentials. Supports local paths and `http(s)` links. | -| **`priority`** | Int | No | `0` | Determines the lookup order. **Lower numbers have higher priority.** For example, `-2` is processed before `0`. Use negative numbers for primary sources. | -| **`logo_override`** | Bool | No | `false` | If set to `true`, channel logos from the provider are replaced by the icons found in the XMLTV file. | +| Parameter | Type | Required | Default | Technical Impact & Background | +| :------------------ | :----- | :------: | :------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`type`** | Enum | No | `xmltv` | Defines the source format. Supported values are `xmltv` and `ics`. Existing configurations without `type` are treated as `xmltv`. | +| **`url`** | String | Yes | | The EPG source URL or local path. For XMLTV sources, use **`auto`** with Xtream inputs to automatically generate the native XMLTV URL using your credentials. For ICS sources, this points to an `.ics` calendar file. | +| **`priority`** | Int | No | `0` | Determines the lookup order. **Lower numbers have higher priority.** For example, `-2` is processed before `0`. Use negative numbers for primary sources. | +| **`logo_override`** | Bool | No | `false` | If set to `true`, channel logos from the provider are replaced by icons found in the EPG source. This mainly applies to XMLTV sources. | +| **`channel_id`** | String | ICS only | | Required for `type: ics`. Defines the generated XMLTV channel ID for the imported calendar. Playlist entries can reference this value through their EPG ID or receive it through Smart Match. | +| **`channel_title`** | String | No | | Optional display name for the generated ICS EPG channel. If omitted, `channel_id` is used as fallback. This value is also used as a Smart Match candidate. | +| **`match_names`** | List | No | `[]` | Optional additional names for matching the generated ICS channel to playlist entries. Useful when the playlist channel name differs from the calendar title, for example `F1`, `Formula One`, or `Formel 1`. | +| **`ics`** | Object | ICS only | | Additional configuration for `type: ics`. See [ICS Calendar Source Parameters](#ics-calendar-source-parameters-ics). | + +#### XMLTV Sources + +XMLTV is the default source type. Existing configurations remain valid and do not need to be changed. + +```yaml +epg: + sources: + - url: "auto" + priority: -2 + logo_override: true + + - url: "https://example.org/xmltv.xml" + priority: 0 +``` + +The following configuration is equivalent: + +```yaml +epg: + sources: + - type: xmltv + url: "https://example.org/xmltv.xml" + priority: 0 +``` + +For Xtream inputs, `url: "auto"` automatically generates the provider's native XMLTV endpoint from the configured +input URL, username, and password. + +The ICS-only fields `channel_id`, `channel_title`, `match_names`, and `ics` are rejected on `type: xmltv` sources. This +keeps accidental calendar settings from changing XMLTV download or runtime behavior. + +#### ICS Calendar Sources + +ICS sources import iCalendar files and convert their `VEVENT` entries into XMLTV-compatible programme entries. +Recurring events using `RRULE`, `RDATE`, or `EXDATE` are detected but not expanded yet; Tuliprox imports the base +`DTSTART`/`DTEND` occurrence and logs one aggregated warning per parsed source when such entries are present. + +An ICS source creates exactly one virtual EPG channel. It does **not** create a playlist channel or stream. The generated +`channel_id` must therefore be assigned to an existing playlist entry by one of the following mechanisms: + +* the playlist entry already uses the same EPG ID, +* a mapper rule assigns the EPG ID, +* Smart Match matches the playlist channel name against `channel_id`, `channel_title`, or `match_names`. + +The generated programmes are written into the regular XMLTV output. M3U and Xtream use the same internal +`epg_channel_id` assignment, so the same `channel_id` works for both input types. If an Xtream provider supplies a valid +but unwanted EPG ID, use a mapping rule to overwrite the stream's EPG ID with the ICS `channel_id`. + +Example using the public Formula 1 calendar: + +```yaml +epg: + sources: + - type: ics + url: "https://files-f1.motorsportcalendars.com/f1-calendar_p1_p2_p3_qualifying_sprint_gp.ics" + channel_id: "f1.calendar" + channel_title: "Formula 1" + priority: -10 + match_names: + - "F1" + - "Formula One" + - "Formel 1" + ics: + timezone: "Europe/Budapest" + event: + title: "{summary}" + description: "{description}" + include_location: true + include_categories: true + dummy: + enabled: true + title: "No Formula 1 session" + description: "There is currently no scheduled Formula 1 session." + days_past: 1 + days_future: 30 + block_hours: 4 + min_gap_minutes: 1 +``` + +A playlist channel can then be linked explicitly by using the generated EPG channel ID: + +```m3u +#EXTINF:-1 tvg-id="f1.calendar" tvg-name="Formula 1",Formula 1 +http://example.org/live/f1/index.m3u8 +``` + +Alternatively, Smart Match can match playlist names such as `Formula 1`, `F1`, or `Formel 1` when the corresponding +values are configured through `channel_title` or `match_names`. + +#### ICS Calendar Source Parameters (`ics`) + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :--------------------------- | :----- | :------: | :--------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`timezone`** | String | No | `UTC` | Fallback timezone for floating ICS timestamps and the local dummy block calculation. Use an IANA zone such as `Europe/Budapest`, `UTC`, or `America/New_York`. | +| **`event`** | Object | No | | Controls how calendar events are mapped to programme title and description. See [ICS Event Mapping](#ics-event-mapping-event). | +| **`dummy`** | Object | No | | Controls optional gap filling with generated dummy programme entries. See [ICS Dummy Gap Filling](#ics-dummy-gap-filling-dummy). | +| **`include_cancelled`** | Bool | No | `false` | If set to `true`, calendar events with `STATUS:CANCELLED` are imported. By default, cancelled events are skipped. | +| **`max_events`** | Int | No | `50000` | Safety budget for encountered `VEVENT` blocks, including invalid or skipped events. Parsing stops when the budget is exhausted. Hard cap: `200000`. | +| **`max_download_bytes`** | Int | No | `10485760` | Maximum downloaded ICS size in bytes. The default is 10 MiB. Hard cap: `52428800` (50 MiB). | +| **`max_decompressed_bytes`** | Int | No | `20971520` | Maximum decompressed ICS size in bytes. The default is 20 MiB. Hard cap: `104857600` (100 MiB). | + +`timezone` must be a valid IANA timezone known to Tuliprox, for example `UTC`, `Europe/Budapest`, or +`America/New_York`. + +#### ICS Event Mapping (`event`) + +The `event` block controls how ICS `VEVENT` properties are converted into EPG programme fields. + +```yaml +ics: + event: + title: "{summary}" + description: "{description}" + include_location: true + include_categories: true +``` + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :----------------------- | :----- | :------: | :-------------- | :-------------------------------------------------------------------------------------- | +| **`title`** | String | No | `{summary}` | Template used for the generated programme title. | +| **`description`** | String | No | `{description}` | Template used for the generated programme description. | +| **`include_location`** | Bool | No | `false` | Appends the ICS `LOCATION` value to the generated programme description when available. | +| **`include_categories`** | Bool | No | `false` | Appends ICS `CATEGORIES` to the generated programme description when available. | + +Supported template variables: + +| Variable | Description | +| :------------------ | :------------------------------------------------ | +| **`{summary}`** | The ICS `SUMMARY` value. Usually the event title. | +| **`{description}`** | The ICS `DESCRIPTION` value. | +| **`{location}`** | The ICS `LOCATION` value. | +| **`{categories}`** | The ICS `CATEGORIES` value. | +| **`{uid}`** | The ICS `UID` value. | +| **`{start}`** | The localized event start time. | +| **`{end}`** | The localized event end time. | + +Example: + +```yaml +ics: + event: + title: "Formula 1: {summary}" + description: "{description}" + include_location: true + include_categories: true +``` + +#### ICS Time Handling + +Tuliprox converts all imported calendar events into the internal EPG time format. + +Supported ICS time forms: + +| ICS Time Form | Behavior | +| :--------------------------------------------- | :----------------------------------------------------------------------- | +| `DTSTART:20260306T123000Z` | Treated as UTC. | +| `DTSTART;TZID=Europe/Berlin:20260306T1230` | Interpreted in the specified `TZID` timezone and converted internally. | +| `DTSTART:20260306T123000` | Treated as a floating timestamp and interpreted using `ics.timezone`. | +| `DTSTART;VALUE=DATE:20260306` | All-day events. These are ignored. | + +For regular programme entries, an ICS event must provide a valid start and end time. `DTEND` is preferred. If `DTEND` +is missing, Tuliprox may use `DURATION` when present. Events without a usable end time are skipped. For template +variables, `{start}` and `{end}` are formatted consistently; when `DURATION` supplies the end time, `{end}` uses the +same display timezone as `DTSTART`. + +#### ICS Dummy Gap Filling (`dummy`) + +ICS calendars often only contain real events, for example race sessions, meetings, or special broadcasts. This may leave +large gaps in the EPG. The optional dummy gap filler can create placeholder programmes for these gaps. + +Dummy entries are generated in local day blocks. By default, the block size is 4 hours: + +```text +00:00 - 04:00 +04:00 - 08:00 +08:00 - 12:00 +12:00 - 16:00 +16:00 - 20:00 +20:00 - 24:00 +``` + +Example: + +```yaml +ics: + timezone: "UTC" + dummy: + enabled: true + title: "No Formula 1 session" + description: "There is currently no scheduled Formula 1 session." + days_past: 1 + days_future: 30 + block_hours: 4 + min_gap_minutes: 1 +``` + +| Parameter | Type | Required | Default | Technical Impact & Background | +| :-------------------- | :----- | :------: | :------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`enabled`** | Bool | No | `false` | Enables generation of dummy programme entries for gaps without real events. | +| **`title`** | String | No | `No programme entry` | Programme title for generated dummy entries. | +| **`description`** | String | No | empty | Programme description for generated dummy entries. | +| **`days_past`** | Int | No | `1` | Number of days before the current day for which dummy entries are generated. | +| **`days_future`** | Int | No | `14` | Number of days after the current day for which dummy entries are generated. | +| **`block_hours`** | Int | No | `4` | Size of the local dummy blocks in hours. The value must divide 24 evenly. For example: `1`, `2`, `3`, `4`, `6`, `8`, or `12`. | +| **`min_gap_minutes`** | Int | No | `1` | Minimum gap length required to generate a dummy entry. Smaller gaps are ignored to avoid tiny placeholder programmes caused by second-level timestamp differences. | + +Dummy entries never replace real programme entries. They only fill uncovered time ranges. + +`days_past` is capped at `30`; `days_future` is capped at `366`. Dummy titles use the same length cap as event summaries +and dummy descriptions use the same length cap as event descriptions. + +Example with one real event from `03:30` to `05:00`: + +```text +00:00 - 03:30 No Formula 1 session +03:30 - 05:00 Real calendar event +05:00 - 08:00 No Formula 1 session +08:00 - 12:00 No Formula 1 session +... +``` + +Example with an event aligned to a block boundary: + +```text +00:00 - 04:00 No Formula 1 session +04:00 - 06:00 Real calendar event +06:00 - 08:00 No Formula 1 session +08:00 - 12:00 No Formula 1 session +... +``` + +If two events are adjacent, no dummy programme is inserted between them: + +```text +10:00 - 11:00 Real calendar event +11:00 - 12:00 Real calendar event +12:00 - 16:00 No Formula 1 session +``` + +Dummy block boundaries are calculated in `ics.timezone`. Around daylight saving time changes, the local block labels +remain stable, for example `00:00 - 04:00`, `04:00 - 08:00`, and so on, while the corresponding UTC duration may differ. +If a local boundary does not exist during a spring-forward transition, it maps to the first real instant after the gap. +If a boundary is ambiguous during a fall-back transition, the earlier occurrence is used. Any resulting zero-length or +negative UTC interval is omitted, so generated dummy programmes remain ordered, non-overlapping, and continuous over +the real local day. + +#### ICS Download and Security Notes + +ICS sources use the same download infrastructure as playlist and XMLTV EPG downloads. This means that input-level +headers, disabled headers, default user-agent handling, cache handling, retry behavior, provider failover, and sanitized +error logging are applied consistently. + +Supported URL forms for ICS sources: + +| URL Form | Description | +| :--------------- | :------------------------------------------------------------------------ | +| `https://...` | Recommended for remote calendar files. | +| `http://...` | Supported when explicitly needed. | +| `webcal://...` | Normalized internally to `https://...`. | +| `provider://...` | Uses a configured Tuliprox provider failover definition. | +| `file://...` | Reads a local calendar file from the host filesystem. | +| local path | Reads a local relative or absolute file path, subject to path validation. | + +Unsupported or unsafe schemes such as `data:`, `ftp:`, `gopher:`, `javascript:`, `mailto:`, or `ssh:` are rejected. + +Tuliprox does not load external resources referenced inside the ICS file. Fields such as `ATTACH`, `URL`, `ORGANIZER`, +`ATTENDEE`, or `IMAGE` are treated as metadata only and must not trigger additional network requests. + +To protect the server, ICS imports are bounded by safety limits such as maximum download size and maximum event count. +Sensitive request data, such as credentials and authorization headers, is sanitized in logs and error messages. + +Additional parser limits are enforced regardless of the configured values: + +| Limit | Hard Cap | +| :---- | :------- | +| Physical or unfolded ICS line length | 128 KiB | +| Properties per `VEVENT` | 256 | +| Imported `SUMMARY` text | 4 KiB, truncated at a UTF-8 boundary | +| Imported `DESCRIPTION` text | 64 KiB, truncated at a UTF-8 boundary | + +If the file itself is unreadable, too large, not valid UTF-8, violates the line-length limit, or does not contain one +complete, correctly nested `VCALENDAR` envelope, the source fails. A correctly wrapped calendar without events is +valid. If a single `VEVENT` inside that envelope is malformed, has too many properties, has an unknown event `TZID`, or +lacks a usable +`DTSTART`/`DTEND`/`DURATION`, that event is skipped and the remaining events continue to be imported. The Web UI EPG +preview uses the same merged output behavior as the client XMLTV output, including ICS dummy gap filling. If a cached +`.ics` file cannot be parsed in preview, Tuliprox ignores the cached file and downloads the source again through the +normal EPG download path. + +The graphical source editor currently preserves existing ICS fields when editing an input, but full creation and editing +of every ICS-specific field is YAML-first. Configure new ICS sources in `source.yml`. #### Smart Match Parameters (`smart_match`) The fuzzy matching logic attempts to "guess" the EPG ID by generating search keys based on the channel name. +For XMLTV sources, Tuliprox uses the channel IDs and display names from the XMLTV file. + +For ICS sources, Tuliprox uses the generated virtual channel metadata: + +* `channel_id` +* `channel_title` +* `match_names` + | Parameter | Type | Default | Technical Impact | -|:------------------------|:-------|:-------------------|:---------------------------------------------------------------------------------------------------------------------| +| :---------------------- | :----- | :----------------- | :------------------------------------------------------------------------------------------------------------------- | | `enabled` | Bool | `false` | Activates the Smart Match engine for streams without a fixed `tvg-id`. | | `fuzzy_matching` | Bool | `false` | Fallback to phonetic and Jaro-Winkler similarity matching if exact ID match fails. | | `match_threshold` | Int | `80` | Minimum similarity score (10-100) required to accept a fuzzy match. | @@ -293,10 +625,33 @@ If a stream is missing the `tvg-id`, Tuliprox performs the following steps: 4. **Reconstruction:** Using `name_prefix.suffix` (`.`), the country code is appended to the name. The target search key becomes `hbo.us`. 5. **Phonetic Matching:** The engine uses **Double Metaphone** phonetic encoding to search for the ID `hbo.us` in the - aggregated XMLTV database. + aggregated EPG database. + +For an ICS source, the generated EPG channel participates in the same matching process. For example, this configuration: + +```yaml +epg: + sources: + - type: ics + url: "https://files-f1.motorsportcalendars.com/f1-calendar_p1_p2_p3_qualifying_sprint_gp.ics" + channel_id: "f1.calendar" + channel_title: "Formula 1" + match_names: + - "F1" + - "Formel 1" +``` + +can match playlist channel names such as: + +```text +Formula 1 +F1 +Formel 1 +``` > **Note:** Lower `match_threshold` values increase the chance of EPG assignment but may lead to incorrect matches for > channels with very similar names. + --- ### 2.4 Provider Aliases (`aliases` & `batch://`) @@ -1148,9 +1503,9 @@ The following attributes are recognized and preserved during M3U import and expo **Proxy Behavior:** -* **Direct Output:** When Tuliprox outputs streams directly (e.g., when not using proxy rewrite), it preserves the original provider +* **Direct Output:** When Tuliprox outputs streams directly (e.g., when not using proxy rewrite), it preserves the original provider metadata exactly as received. -* **Rewritten Output (Reverse Proxy):** When Tuliprox rewrites URLs to act as a reverse proxy, it generates authenticated local Tuliprox +* **Rewritten Output (Reverse Proxy):** When Tuliprox rewrites URLs to act as a reverse proxy, it generates authenticated local Tuliprox archive URLs instead of leaking the provider's upstream archive URLs. **Supported Modes:** diff --git a/frontend/src/app/components/source_editor/epg_source_item_form.rs b/frontend/src/app/components/source_editor/epg_source_item_form.rs index 07a30c18a..db17cdb38 100644 --- a/frontend/src/app/components/source_editor/epg_source_item_form.rs +++ b/frontend/src/app/components/source_editor/epg_source_item_form.rs @@ -35,14 +35,8 @@ pub struct EpgSourceItemFormProps { pub fn EpgSourceItemForm(props: &EpgSourceItemFormProps) -> Html { let translate = use_translation(); - let form_state: UseReducerHandle = use_reducer(|| EpgSourceFormState { - form: props.initial.clone().unwrap_or_else(|| EpgSourceDto { - url: String::new(), - priority: 0, - logo_override: false, - }), - modified: false, - }); + let form_state: UseReducerHandle = + use_reducer(|| EpgSourceFormState { form: props.initial.clone().unwrap_or_default(), modified: false }); let handle_submit = { let form_state = form_state.clone(); diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 024754e23..6420ee150 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -23,6 +23,7 @@ blake3.workspace = true fastrand = { workspace = true, features = ["js"] } zeroize.workspace = true chrono.workspace = true +chrono-tz = "0.10.4" bytes.workspace = true lz4_flex.workspace = true brotli.workspace = true diff --git a/shared/src/defaults/epg.rs b/shared/src/defaults/epg.rs index 1fc8b26ee..a39d6081d 100644 --- a/shared/src/defaults/epg.rs +++ b/shared/src/defaults/epg.rs @@ -5,6 +5,43 @@ default_eq_fns!( default_epg_best_match_threshold, is_default_epg_best_match_threshold, u16, 95; ); +default_eq_fns!( + default_ics_dummy_days_past, is_default_ics_dummy_days_past, u16, 1; + default_ics_dummy_days_future, is_default_ics_dummy_days_future, u16, 14; + default_ics_dummy_block_hours, is_default_ics_dummy_block_hours, u8, 4; + default_ics_dummy_min_gap_minutes, is_default_ics_dummy_min_gap_minutes, u16, 1; + default_ics_max_events, is_default_ics_max_events, usize, 50_000; + default_ics_max_download_bytes, is_default_ics_max_download_bytes, u64, 10 * 1024 * 1024; + default_ics_max_decompressed_bytes, is_default_ics_max_decompressed_bytes, usize, 20 * 1024 * 1024; +); + +pub const MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT: u64 = 50 * 1024 * 1024; +pub const MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT: usize = 100 * 1024 * 1024; +pub const MAX_ICS_EVENTS_HARD_LIMIT: usize = 200_000; +pub const MAX_ICS_LINE_LENGTH: usize = 128 * 1024; +pub const MAX_ICS_PROPERTIES_PER_EVENT: usize = 256; +pub const MAX_ICS_SUMMARY_LENGTH: usize = 4 * 1024; +pub const MAX_ICS_DESCRIPTION_LENGTH: usize = 64 * 1024; +pub const MAX_ICS_DAYS_PAST: u16 = 30; +pub const MAX_ICS_DAYS_FUTURE: u16 = 366; + +pub const DEFAULT_ICS_TIMEZONE: &str = "UTC"; +pub const DEFAULT_ICS_EVENT_TITLE: &str = "{summary}"; +pub const DEFAULT_ICS_EVENT_DESCRIPTION: &str = "{description}"; +pub const DEFAULT_ICS_DUMMY_TITLE: &str = "No programme entry"; + +pub fn default_ics_timezone() -> String { DEFAULT_ICS_TIMEZONE.to_string() } +pub fn is_default_ics_timezone(value: &String) -> bool { value == DEFAULT_ICS_TIMEZONE } + +pub fn default_ics_event_title() -> String { DEFAULT_ICS_EVENT_TITLE.to_string() } +pub fn is_default_ics_event_title(value: &String) -> bool { value == DEFAULT_ICS_EVENT_TITLE } + +pub fn default_ics_event_description() -> String { DEFAULT_ICS_EVENT_DESCRIPTION.to_string() } +pub fn is_default_ics_event_description(value: &String) -> bool { value == DEFAULT_ICS_EVENT_DESCRIPTION } + +pub fn default_ics_dummy_title() -> String { DEFAULT_ICS_DUMMY_TITLE.to_string() } +pub fn is_default_ics_dummy_title(value: &String) -> bool { value == DEFAULT_ICS_DUMMY_TITLE } + pub const DEFAULT_EPG_NORMALIZE_REGEX: &str = r"[^a-zA-Z0-9\-]"; pub fn default_epg_normalize_regex() -> Option { Some(DEFAULT_EPG_NORMALIZE_REGEX.to_string()) } diff --git a/shared/src/model/config/epg.rs b/shared/src/model/config/epg.rs index 8ce929f27..63df9020f 100644 --- a/shared/src/model/config/epg.rs +++ b/shared/src/model/config/epg.rs @@ -1,21 +1,237 @@ -use crate::{defaults::is_false, error::TuliproxError, model::EpgSmartMatchConfigDto}; +use crate::{ + defaults::{ + default_ics_dummy_block_hours, default_ics_dummy_days_future, default_ics_dummy_days_past, + default_ics_dummy_min_gap_minutes, default_ics_dummy_title, default_ics_event_description, + default_ics_event_title, default_ics_max_decompressed_bytes, default_ics_max_download_bytes, + default_ics_max_events, default_ics_timezone, is_default_ics_dummy_block_hours, + is_default_ics_dummy_days_future, is_default_ics_dummy_days_past, is_default_ics_dummy_min_gap_minutes, + is_default_ics_dummy_title, is_default_ics_event_description, is_default_ics_event_title, + is_default_ics_max_decompressed_bytes, is_default_ics_max_download_bytes, is_default_ics_max_events, + is_default_ics_timezone, is_false, MAX_ICS_DAYS_FUTURE, MAX_ICS_DAYS_PAST, + MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT, MAX_ICS_DESCRIPTION_LENGTH, MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT, + MAX_ICS_EVENTS_HARD_LIMIT, MAX_ICS_SUMMARY_LENGTH, + }, + error::TuliproxError, + model::EpgSmartMatchConfigDto, + utils::{is_blank_optional_string, sanitize_sensitive_info}, +}; const AUTO_URL: &str = "auto"; +#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum EpgSourceTypeDto { + #[default] + Xmltv, + Ics, +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct EpgSourceDto { + #[serde(default, rename = "type")] + pub source_type: EpgSourceTypeDto, pub url: String, #[serde(default)] pub priority: i16, #[serde(default, skip_serializing_if = "is_false")] pub logo_override: bool, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub channel_id: Option, + #[serde(default, skip_serializing_if = "is_blank_optional_string")] + pub channel_title: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub match_names: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ics: Option, +} + +impl Default for EpgSourceDto { + fn default() -> Self { + Self { + source_type: EpgSourceTypeDto::Xmltv, + url: String::new(), + priority: 0, + logo_override: false, + channel_id: None, + channel_title: None, + match_names: Vec::new(), + ics: None, + } + } } impl EpgSourceDto { - pub fn prepare(&mut self) { self.url = self.url.trim().to_string(); } + pub fn prepare(&mut self) -> Result<(), TuliproxError> { + self.url = self.url.trim().to_string(); + self.channel_id = + self.channel_id.take().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + self.channel_title = + self.channel_title.take().map(|value| value.trim().to_string()).filter(|value| !value.is_empty()); + self.match_names = self + .match_names + .drain(..) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .collect(); + + match self.source_type { + EpgSourceTypeDto::Xmltv => self.prepare_xmltv(), + EpgSourceTypeDto::Ics => self.prepare_ics(), + } + } + + fn prepare_xmltv(&self) -> Result<(), TuliproxError> { + if self.url.is_empty() { + return Err(TuliproxError::ConfigEpg("XMLTV EPG source url is empty".to_string())); + } + if self.channel_id.is_some() + || self.channel_title.is_some() + || !self.match_names.is_empty() + || self.ics.is_some() + { + return Err(TuliproxError::ConfigEpg( + "channel_id, channel_title, match_names, and ics are only supported for ICS EPG sources".to_string(), + )); + } + Ok(()) + } + + fn prepare_ics(&mut self) -> Result<(), TuliproxError> { + if self.url.is_empty() { + return Err(TuliproxError::ConfigEpg("ICS EPG source url is empty".to_string())); + } + if self.url.eq_ignore_ascii_case(AUTO_URL) { + return Err(TuliproxError::ConfigEpg( + "url: auto is only supported for XMLTV EPG sources, not for ICS".to_string(), + )); + } + if let Some(rest) = strip_prefix_ignore_ascii_case(&self.url, "webcal://") { + self.url = format!("https://{rest}"); + } + if self.channel_id.as_deref().is_none_or(str::is_empty) { + return Err(TuliproxError::ConfigEpg("channel_id is required for ICS EPG sources".to_string())); + } + let ics = self.ics.get_or_insert_with(IcsEpgSourceConfigDto::default); + ics.timezone = ics.timezone.trim().to_string(); + validate_ics_config(ics)?; + validate_ics_url_scheme(&self.url) + } pub fn is_valid(&self) -> bool { !self.url.is_empty() } + + pub fn source_identity(&self) -> String { + match self.source_type { + EpgSourceTypeDto::Xmltv => format!("xmltv|{}", self.url), + EpgSourceTypeDto::Ics => { + format!("ics|{}|{}", self.url, self.channel_id.as_deref().unwrap_or_default()) + } + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct IcsEpgSourceConfigDto { + #[serde(default = "default_ics_timezone", skip_serializing_if = "is_default_ics_timezone")] + pub timezone: String, + #[serde(default, skip_serializing_if = "IcsEventMappingDto::is_default")] + pub event: IcsEventMappingDto, + #[serde(default, skip_serializing_if = "IcsDummyConfigDto::is_default")] + pub dummy: IcsDummyConfigDto, + #[serde(default, skip_serializing_if = "is_false")] + pub include_cancelled: bool, + #[serde(default = "default_ics_max_events", skip_serializing_if = "is_default_ics_max_events")] + pub max_events: usize, + #[serde(default = "default_ics_max_download_bytes", skip_serializing_if = "is_default_ics_max_download_bytes")] + pub max_download_bytes: u64, + #[serde( + default = "default_ics_max_decompressed_bytes", + skip_serializing_if = "is_default_ics_max_decompressed_bytes" + )] + pub max_decompressed_bytes: usize, +} + +impl Default for IcsEpgSourceConfigDto { + fn default() -> Self { + Self { + timezone: default_ics_timezone(), + event: IcsEventMappingDto::default(), + dummy: IcsDummyConfigDto::default(), + include_cancelled: false, + max_events: default_ics_max_events(), + max_download_bytes: default_ics_max_download_bytes(), + max_decompressed_bytes: default_ics_max_decompressed_bytes(), + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct IcsEventMappingDto { + #[serde(default = "default_ics_event_title", skip_serializing_if = "is_default_ics_event_title")] + pub title: String, + #[serde(default = "default_ics_event_description", skip_serializing_if = "is_default_ics_event_description")] + pub description: String, + #[serde(default, skip_serializing_if = "is_false")] + pub include_location: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub include_categories: bool, +} + +impl Default for IcsEventMappingDto { + fn default() -> Self { + Self { + title: default_ics_event_title(), + description: default_ics_event_description(), + include_location: false, + include_categories: false, + } + } +} + +impl IcsEventMappingDto { + pub fn is_default(value: &Self) -> bool { value == &Self::default() } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct IcsDummyConfigDto { + #[serde(default, skip_serializing_if = "is_false")] + pub enabled: bool, + #[serde(default = "default_ics_dummy_title", skip_serializing_if = "is_default_ics_dummy_title")] + pub title: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub description: String, + #[serde(default = "default_ics_dummy_days_past", skip_serializing_if = "is_default_ics_dummy_days_past")] + pub days_past: u16, + #[serde(default = "default_ics_dummy_days_future", skip_serializing_if = "is_default_ics_dummy_days_future")] + pub days_future: u16, + #[serde(default = "default_ics_dummy_block_hours", skip_serializing_if = "is_default_ics_dummy_block_hours")] + pub block_hours: u8, + #[serde( + default = "default_ics_dummy_min_gap_minutes", + skip_serializing_if = "is_default_ics_dummy_min_gap_minutes" + )] + pub min_gap_minutes: u16, +} + +impl Default for IcsDummyConfigDto { + fn default() -> Self { + Self { + enabled: false, + title: default_ics_dummy_title(), + description: String::new(), + days_past: default_ics_dummy_days_past(), + days_future: default_ics_dummy_days_future(), + block_hours: default_ics_dummy_block_hours(), + min_gap_minutes: default_ics_dummy_min_gap_minutes(), + } + } +} + +impl IcsDummyConfigDto { + pub fn is_default(value: &Self) -> bool { value == &Self::default() } } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Default)] @@ -32,10 +248,8 @@ pub struct EpgConfigDto { impl EpgConfigDto { /// Prepares the EPG configuration by resolving all source URLs into `t_sources`. /// - /// - `create_auto_url` — closure that derives an XMLTV URL from the parent input - /// (called when `url` is `auto`). - /// - `include_computed` — when `false` the resolution is skipped (used for serialisation - /// round-trips that do not need fully-resolved URLs). + /// - `create_auto_url` derives an XMLTV URL from the parent input when XMLTV `url` is `auto`. + /// - `include_computed` skips resolution when serialisation round-trips do not need computed URLs. pub fn prepare(&mut self, create_auto_url: F, include_computed: bool) -> Result<(), TuliproxError> where F: Fn() -> Result, @@ -44,19 +258,19 @@ impl EpgConfigDto { self.t_sources = Vec::new(); if let Some(epg_sources) = self.sources.as_mut() { for epg_source in epg_sources.iter_mut() { - epg_source.prepare(); + epg_source.prepare()?; if !epg_source.is_valid() { continue; } - if epg_source.url.eq_ignore_ascii_case(AUTO_URL) { + if epg_source.source_type == EpgSourceTypeDto::Xmltv + && epg_source.url.eq_ignore_ascii_case(AUTO_URL) + { match create_auto_url() { Ok(provider_url) => { - self.t_sources.push(EpgSourceDto { - url: provider_url, - priority: epg_source.priority, - logo_override: epg_source.logo_override, - }); + let mut resolved = epg_source.clone(); + resolved.url = provider_url; + self.t_sources.push(resolved); } Err(err) => return Err(TuliproxError::ConfigEpg(err.to_string())), } @@ -74,35 +288,134 @@ impl EpgConfigDto { } } +fn validate_ics_config(config: &IcsEpgSourceConfigDto) -> Result<(), TuliproxError> { + validate_ics_timezone(&config.timezone)?; + let block_hours = config.dummy.block_hours; + if block_hours == 0 || block_hours > 24 || 24 % block_hours != 0 { + return Err(TuliproxError::ConfigEpg(format!( + "ics.dummy.block_hours must divide 24 evenly, got {block_hours}" + ))); + } + if config.max_events == 0 { + return Err(TuliproxError::ConfigEpg("ics.max_events must be greater than 0".to_string())); + } + if config.max_events > MAX_ICS_EVENTS_HARD_LIMIT { + return Err(TuliproxError::ConfigEpg(format!("ics.max_events must not exceed {MAX_ICS_EVENTS_HARD_LIMIT}"))); + } + if config.max_download_bytes == 0 { + return Err(TuliproxError::ConfigEpg("ics.max_download_bytes must be greater than 0".to_string())); + } + if config.max_download_bytes > MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT { + return Err(TuliproxError::ConfigEpg(format!( + "ics.max_download_bytes must not exceed {MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT}" + ))); + } + if config.max_decompressed_bytes == 0 { + return Err(TuliproxError::ConfigEpg("ics.max_decompressed_bytes must be greater than 0".to_string())); + } + if config.max_decompressed_bytes > MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT { + return Err(TuliproxError::ConfigEpg(format!( + "ics.max_decompressed_bytes must not exceed {MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT}" + ))); + } + if config.dummy.days_past > MAX_ICS_DAYS_PAST { + return Err(TuliproxError::ConfigEpg(format!("ics.dummy.days_past must not exceed {MAX_ICS_DAYS_PAST}"))); + } + if config.dummy.days_future > MAX_ICS_DAYS_FUTURE { + return Err(TuliproxError::ConfigEpg(format!("ics.dummy.days_future must not exceed {MAX_ICS_DAYS_FUTURE}"))); + } + validate_text_limit("ics.event.title", &config.event.title, MAX_ICS_SUMMARY_LENGTH)?; + validate_text_limit("ics.event.description", &config.event.description, MAX_ICS_DESCRIPTION_LENGTH)?; + validate_text_limit("ics.dummy.title", &config.dummy.title, MAX_ICS_SUMMARY_LENGTH)?; + validate_text_limit("ics.dummy.description", &config.dummy.description, MAX_ICS_DESCRIPTION_LENGTH)?; + Ok(()) +} + +fn validate_ics_timezone(timezone: &str) -> Result<(), TuliproxError> { + if timezone.is_empty() { + return Err(TuliproxError::ConfigEpg("ics.timezone must not be empty".to_string())); + } + timezone + .parse::() + .map(|_| ()) + .map_err(|_| TuliproxError::ConfigEpg(format!("ics.timezone '{timezone}' is not a valid IANA timezone"))) +} + +fn validate_text_limit(field: &str, value: &str, max_len: usize) -> Result<(), TuliproxError> { + if value.len() > max_len { + return Err(TuliproxError::ConfigEpg(format!("{field} must not exceed {max_len} bytes"))); + } + Ok(()) +} + +fn validate_ics_url_scheme(url: &str) -> Result<(), TuliproxError> { + if is_absolute_local_path(url) { + return Ok(()); + } + + let Ok(parsed) = url::Url::parse(url) else { + return Ok(()); + }; + + match parsed.scheme() { + "https" | "http" | "file" | "provider" => Ok(()), + scheme => Err(TuliproxError::ConfigEpg(format!( + "Unsupported ICS url scheme '{scheme}' for {}", + sanitize_sensitive_info(url) + ))), + } +} + +fn strip_prefix_ignore_ascii_case<'a>(value: &'a str, prefix: &str) -> Option<&'a str> { + let candidate = value.get(..prefix.len())?; + if !candidate.eq_ignore_ascii_case(prefix) { + return None; + } + value.get(prefix.len()..) +} + +fn is_absolute_local_path(value: &str) -> bool { + if std::path::Path::new(value).is_absolute() { + return true; + } + + let bytes = value.as_bytes(); + let windows_drive_path = + bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && matches!(bytes[2], b'\\' | b'/'); + windows_drive_path || bytes.starts_with(b"\\\\") +} + #[cfg(test)] mod tests { use super::*; + fn xmltv_source(url: &str) -> EpgSourceDto { EpgSourceDto { url: url.to_owned(), ..EpgSourceDto::default() } } + + fn ics_source(url: &str, channel_id: Option<&str>) -> EpgSourceDto { + EpgSourceDto { + source_type: EpgSourceTypeDto::Ics, + url: url.to_owned(), + channel_id: channel_id.map(ToOwned::to_owned), + ..EpgSourceDto::default() + } + } + #[test] - fn test_plain_url_passthrough() { - let mut cfg = EpgConfigDto { - sources: Some(vec![EpgSourceDto { - url: "http://example.com/xmltv.php".to_owned(), - priority: 0, - logo_override: false, - }]), - ..Default::default() - }; + fn xmltv_source_without_type_remains_valid() { + let mut cfg = + EpgConfigDto { sources: Some(vec![xmltv_source("http://example.com/xmltv.php")]), ..Default::default() }; cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); assert_eq!(cfg.t_sources.len(), 1); + assert_eq!(cfg.t_sources[0].source_type, EpgSourceTypeDto::Xmltv); assert_eq!(cfg.t_sources[0].url, "http://example.com/xmltv.php"); } #[test] - fn test_provider_scheme_kept_unresolved() { - let mut cfg = EpgConfigDto { - sources: Some(vec![EpgSourceDto { - url: "provider://myprovider/xmltv.php?username=u&password=p".to_owned(), - priority: 1, - logo_override: true, - }]), - ..Default::default() - }; + fn provider_scheme_kept_unresolved() { + let mut source = xmltv_source("provider://myprovider/xmltv.php?username=u&password=p"); + source.priority = 1; + source.logo_override = true; + let mut cfg = EpgConfigDto { sources: Some(vec![source]), ..Default::default() }; cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); assert_eq!(cfg.t_sources.len(), 1); assert_eq!(cfg.t_sources[0].url, "provider://myprovider/xmltv.php?username=u&password=p"); @@ -111,11 +424,8 @@ mod tests { } #[test] - fn test_auto_url_used() { - let mut cfg = EpgConfigDto { - sources: Some(vec![EpgSourceDto { url: AUTO_URL.to_owned(), priority: 0, logo_override: false }]), - ..Default::default() - }; + fn xmltv_auto_url_used() { + let mut cfg = EpgConfigDto { sources: Some(vec![xmltv_source(AUTO_URL)]), ..Default::default() }; cfg.prepare(|| Ok("http://auto.example.com/xmltv.php?username=u&password=p".to_owned()), true) .expect("prepare failed"); assert_eq!(cfg.t_sources.len(), 1); @@ -123,16 +433,210 @@ mod tests { } #[test] - fn test_include_computed_false_skips_resolution() { - let mut cfg = EpgConfigDto { - sources: Some(vec![EpgSourceDto { - url: "provider://myprovider/xmltv.php".to_owned(), - priority: 0, - logo_override: false, - }]), - ..Default::default() - }; + fn include_computed_false_skips_resolution() { + let mut cfg = + EpgConfigDto { sources: Some(vec![xmltv_source("provider://myprovider/xmltv.php")]), ..Default::default() }; cfg.prepare(|| Err("no auto".to_owned()), false).expect("prepare with include_computed=false should succeed"); assert!(cfg.t_sources.is_empty()); } + + #[test] + fn ics_auto_url_is_rejected() { + let mut cfg = + EpgConfigDto { sources: Some(vec![ics_source(AUTO_URL, Some("f1.calendar"))]), ..Default::default() }; + let err = cfg.prepare(|| Ok("http://auto.example.com/xmltv.php".to_owned()), true).unwrap_err(); + assert!(err.to_string().contains("only supported for XMLTV")); + } + + #[test] + fn ics_requires_channel_id() { + let mut cfg = + EpgConfigDto { sources: Some(vec![ics_source("https://example.com/f1.ics", None)]), ..Default::default() }; + let err = cfg.prepare(|| Err("no auto".to_owned()), true).unwrap_err(); + assert!(err.to_string().contains("channel_id is required")); + } + + #[test] + fn ics_webcal_url_is_normalized_to_https() { + for url in ["webcal://example.com/f1.ics", "WEBcal://example.com/f1.ics"] { + let mut cfg = + EpgConfigDto { sources: Some(vec![ics_source(url, Some("f1.calendar"))]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); + assert_eq!(cfg.t_sources[0].url, "https://example.com/f1.ics"); + } + } + + #[test] + fn ics_unicode_local_path_is_accepted_without_panicking() { + let mut source = ics_source("äääää/calendar.ics", Some("calendar")); + source.prepare().expect("unicode local path should be valid"); + } + + #[test] + fn ics_absolute_windows_paths_are_local_sources() { + for path in [r"C:\calendar.ics", "D:/calendar.ics", r"\\server\share\calendar.ics"] { + let mut source = ics_source(path, Some("calendar")); + source.prepare().expect("absolute Windows path should be valid"); + } + } + + #[test] + fn xmltv_rejects_ics_only_fields() { + let mut with_ics_block = xmltv_source("https://example.com/epg.xml"); + with_ics_block.ics = Some(IcsEpgSourceConfigDto::default()); + let err = with_ics_block.prepare().unwrap_err(); + assert!(err.to_string().contains("only supported for ICS")); + + let mut with_channel_id = xmltv_source("https://example.com/epg.xml"); + with_channel_id.channel_id = Some("not-an-xmltv-option".to_string()); + let err = with_channel_id.prepare().unwrap_err(); + assert!(err.to_string().contains("only supported for ICS")); + } + + #[test] + fn ics_unsupported_scheme_is_rejected() { + let mut cfg = EpgConfigDto { + sources: Some(vec![ics_source("ftp://example.com/f1.ics", Some("f1.calendar"))]), + ..Default::default() + }; + let err = cfg.prepare(|| Err("no auto".to_owned()), true).unwrap_err(); + assert!(err.to_string().contains("Unsupported ICS url scheme")); + } + + #[test] + fn ics_match_names_are_trimmed_and_empty_values_removed() { + let mut source = ics_source("https://example.com/f1.ics", Some(" f1.calendar ")); + source.channel_title = Some(" Formula 1 ".to_owned()); + source.match_names = vec![" F1 ".to_owned(), String::new(), " ".to_owned(), "Formel 1".to_owned()]; + let mut cfg = EpgConfigDto { sources: Some(vec![source]), ..Default::default() }; + cfg.prepare(|| Err("no auto".to_owned()), true).expect("prepare failed"); + assert_eq!(cfg.t_sources[0].channel_id.as_deref(), Some("f1.calendar")); + assert_eq!(cfg.t_sources[0].channel_title.as_deref(), Some("Formula 1")); + assert_eq!(cfg.t_sources[0].match_names, vec!["F1", "Formel 1"]); + } + + #[test] + fn aliases_field_is_rejected() { + let yaml = r#" +type: ics +url: https://example.com/f1.ics +channel_id: f1.calendar +aliases: + - F1 +"#; + let err = serde_saphyr::from_str::(yaml).unwrap_err(); + assert!(err.to_string().contains("aliases") || err.to_string().contains("unknown field")); + } + + #[test] + fn ics_block_hours_must_divide_day_evenly() { + let mut valid = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + valid.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { block_hours: 4, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + valid.prepare().expect("valid block size"); + + let mut invalid = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + invalid.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { block_hours: 5, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = invalid.prepare().unwrap_err(); + assert!(err.to_string().contains("must divide 24 evenly")); + } + + #[test] + fn invalid_ics_timezone_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = + Some(IcsEpgSourceConfigDto { timezone: "Mars/Olympus".to_string(), ..IcsEpgSourceConfigDto::default() }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.timezone")); + } + + #[test] + fn ics_max_download_bytes_above_hard_cap_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + max_download_bytes: MAX_ICS_DOWNLOAD_BYTES_HARD_LIMIT + 1, + ..IcsEpgSourceConfigDto::default() + }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.max_download_bytes")); + } + + #[test] + fn ics_max_decompressed_bytes_above_hard_cap_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + max_decompressed_bytes: MAX_ICS_DECOMPRESSED_BYTES_HARD_LIMIT + 1, + ..IcsEpgSourceConfigDto::default() + }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.max_decompressed_bytes")); + } + + #[test] + fn ics_max_events_above_hard_cap_is_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + max_events: MAX_ICS_EVENTS_HARD_LIMIT + 1, + ..IcsEpgSourceConfigDto::default() + }); + + let err = source.prepare().unwrap_err(); + + assert!(err.to_string().contains("ics.max_events")); + } + + #[test] + fn ics_dummy_days_above_hard_caps_are_rejected() { + let mut too_much_past = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + too_much_past.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { days_past: MAX_ICS_DAYS_PAST + 1, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = too_much_past.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.dummy.days_past")); + + let mut too_much_future = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + too_much_future.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { days_future: MAX_ICS_DAYS_FUTURE + 1, ..IcsDummyConfigDto::default() }, + ..IcsEpgSourceConfigDto::default() + }); + let err = too_much_future.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.dummy.days_future")); + } + + #[test] + fn ics_extreme_template_and_dummy_texts_are_rejected() { + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + event: IcsEventMappingDto { + title: "x".repeat(MAX_ICS_SUMMARY_LENGTH + 1), + ..IcsEventMappingDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let err = source.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.event.title")); + + let mut source = ics_source("https://example.com/f1.ics", Some("f1.calendar")); + source.ics = Some(IcsEpgSourceConfigDto { + dummy: IcsDummyConfigDto { + description: "x".repeat(MAX_ICS_DESCRIPTION_LENGTH + 1), + ..IcsDummyConfigDto::default() + }, + ..IcsEpgSourceConfigDto::default() + }); + let err = source.prepare().unwrap_err(); + assert!(err.to_string().contains("ics.dummy.description")); + } } diff --git a/shared/src/model/config/input.rs b/shared/src/model/config/input.rs index a2a8e649e..a3bbaaa7b 100644 --- a/shared/src/model/config/input.rs +++ b/shared/src/model/config/input.rs @@ -898,8 +898,8 @@ impl ConfigInputDto { epg.prepare(|| self.generate_auto_epg_url(), include_computed)?; epg.t_sources = { - let mut seen_urls = HashSet::new(); - epg.t_sources.drain(..).filter(|src| seen_urls.insert(src.url.clone())).collect() + let mut seen_sources = HashSet::new(); + epg.t_sources.drain(..).filter(|src| seen_sources.insert(src.source_identity())).collect() }; self.epg = Some(epg); }