diff --git a/CHANGELOG.md b/CHANGELOG.md index c8165615d..c69fe2293 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -385,6 +385,16 @@ ## ⚙️ New Settings +- **source.yml (target `options.epg_output`)**: + - Added optional `lowercase_ids` (`bool`, default `false`) to canonicalize technical EPG IDs with ASCII lowercase + consistently across visible M3U `tvg-id`, Xtream `epg_channel_id`, XMLTV `` / `` + references, EPG API responses, and target EPG storage keys after a full target refresh. Disabled targets retain + their existing source-case storage keys and ordering. + - Added optional `lowercase_xmltv_display_names` (`bool`, default `false`) to lowercase only XMLTV `` + values during serialization; playlist names and programme metadata remain unchanged, and no persisted rebuild is + normally required. + - Both options are disabled by default, so existing visible outputs remain unchanged. Changing `lowercase_ids` + requires a full target refresh, and clients may need to re-index EPG data once after visible IDs change. - **config.yml (main)**: - Added `interner_gc_interval_secs`: interval in seconds between background string interner GC checks. - Added `interner_gc_min_pool_size`: minimum interned-string pool size required before background interner GC runs. diff --git a/backend/src/api/endpoints/xmltv_api.rs b/backend/src/api/endpoints/xmltv_api.rs index c3095341d..eb9fbb043 100644 --- a/backend/src/api/endpoints/xmltv_api.rs +++ b/backend/src/api/endpoints/xmltv_api.rs @@ -11,12 +11,14 @@ use crate::{ auth::Fingerprint, model::{Config, ConfigTarget, ProxyUserCredentials, EPG_ATTRIB_ID, EPG_TAG_CHANNEL}, repository::{ - epg_query_channels, get_target_storage_path, m3u_get_epg_file_path_for_target, storage_const, - xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTreeQuery, LockedReceiverStream, XML_PREAMBLE, + epg_query_channels_by_storage_key, get_target_storage_path, m3u_get_epg_file_path_for_target, + storage_const, xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTreeQuery, + LockedReceiverStream, XML_PREAMBLE, }, utils, utils::{ - deobscure_text, file_exists_async, format_xmltv_time_utc, get_epg_processing_options, obscure_text, + canonicalize_output_epg_id, canonicalize_untrusted_epg_id, deobscure_text, file_exists_async, + format_xmltv_time_utc, get_epg_processing_options, lowercase_xmltv_text, obscure_text, EpgIdOutputCase, EpgProcessingOptions, EpgTimeShift, }, }; @@ -27,16 +29,12 @@ use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; use shared::{ concat_string, model::{ - EpgChannel, EpgProgramme, EpgProgrammeDto, ShortEpgDto, ShortEpgResultDto, StreamEpgEntry, StreamEpgItemRequest, - StreamEpgRequest, StreamEpgResponse, TargetType, + ConfigTargetOptions, EpgChannel, EpgProgramme, EpgProgrammeDto, ShortEpgDto, ShortEpgResultDto, StreamEpgEntry, + StreamEpgItemRequest, StreamEpgRequest, StreamEpgResponse, TargetType, }, utils::{concat_path, concat_path_leading_slash, obfuscate_text, Internable}, }; -use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, - sync::Arc, -}; +use std::{collections::HashMap, path::{Path, PathBuf}, sync::Arc}; use tokio::{io::AsyncWriteExt, sync::mpsc, task}; use tokio_stream::StreamExt; use tokio_util::io::ReaderStream; @@ -196,6 +194,10 @@ async fn serve_epg_with_rewrites( } let epg_processing_options = get_epg_processing_options(app_state, user, target); + let lowercase_display_names = target + .options + .as_ref() + .is_some_and(ConfigTargetOptions::lowercase_xmltv_display_names); let server_info = app_state.app_config.get_user_server_info(user); let base_url = @@ -270,7 +272,10 @@ async fn serve_epg_with_rewrites( let elem = BytesStart::new("display-name"); continue_on_err!(writer.write_event_async(Event::Start(elem)).await); let title: &str = channel.title.as_deref().unwrap_or(""); - continue_on_err!(writer.write_event_async(Event::Text(BytesText::new(title))).await); + let display_name = lowercase_xmltv_text(title, lowercase_display_names); + continue_on_err!(writer + .write_event_async(Event::Text(BytesText::new(display_name.as_ref()))) + .await); let elem = BytesEnd::new("display-name"); continue_on_err!(writer.write_event_async(Event::End(elem)).await); @@ -348,15 +353,20 @@ async fn serve_epg_with_rewrites( .body(axum::body::Body::from_stream(body_stream))) } -async fn get_epg_channel(app_state: &Arc, channel_id: &Arc, epg_path: &Path) -> Option { +/// Looks up an EPG channel by its exact, target-output-case storage key. +async fn get_epg_channel_by_storage_key( + app_state: &Arc, + storage_key: &Arc, + epg_path: &Path, +) -> Option { let file_lock = app_state.app_config.file_locks.read_lock(epg_path).await; let epg_path = epg_path.to_path_buf(); - let channel_id = Arc::clone(channel_id); + let storage_key = Arc::clone(storage_key); match task::spawn_blocking(move || -> Option { let _guard = file_lock; match BPlusTreeQuery::, EpgChannel>::try_new(&epg_path) { - Ok(mut query) => match query.query(&channel_id) { + Ok(mut query) => match query.query(&storage_key) { Ok(Some(item)) => Some(item), Ok(None) => None, Err(err) => { @@ -473,24 +483,31 @@ pub async fn serve_short_epg( stream_id: Arc, limit: u32, ) -> axum::response::Response { + let lowercase_ids = target + .options + .as_ref() + .is_some_and(ConfigTargetOptions::lowercase_epg_ids); + let output_case = EpgIdOutputCase::from_lowercase(lowercase_ids); + let storage_key = canonicalize_output_epg_id(channel_id, output_case); + let response_channel_id = Arc::clone(&storage_key); let short_epg = { // It seems provider set limit to 4 if it is undefined oor 0. let limit = if limit > 0 { limit } else { DEFAULT_SHORT_EPG_LIMIT }; if file_exists_async(epg_path).await { - if let Some(epg_channel) = get_epg_channel(app_state, channel_id, epg_path).await { + if let Some(epg_channel) = get_epg_channel_by_storage_key(app_state, &storage_key, epg_path).await { let epg_processing_options = get_epg_processing_options(app_state, user, target); ShortEpgResultDto { epg_listings: if limit > 0 { epg_channel .get_programme_with_limit(limit) .iter() - .map(|p| from_programme(&stream_id, channel_id, p, &epg_processing_options)) + .map(|p| from_programme(&stream_id, &response_channel_id, p, &epg_processing_options)) .collect() } else { epg_channel .programmes .iter() - .map(|p| from_programme(&stream_id, channel_id, p, &epg_processing_options)) + .map(|p| from_programme(&stream_id, &response_channel_id, p, &epg_processing_options)) .collect() }, } @@ -516,15 +533,38 @@ pub async fn serve_short_epg( /// Returns empty entries for all error/missing cases (no errors surfaced to client). const STREAM_EPG_ARCHIVE_WINDOW_BACK_SECS: i64 = 4 * 3600; const STREAM_EPG_ARCHIVE_WINDOW_FWD_SECS: i64 = 12 * 3600; +const MAX_STREAM_EPG_ITEMS: usize = 256; +const MAX_STREAM_EPG_CHANNEL_ID_BYTES: usize = 512; + +struct PreparedStreamEpgEntry { + storage_key: Arc, + reference_ts: Option, +} + +struct PreparedStreamEpgRequest { + entries: Vec, +} + +fn prepare_stream_epg_request( + items: &[StreamEpgItemRequest], + output_case: EpgIdOutputCase, +) -> PreparedStreamEpgRequest { + let mut entry_index_by_storage_key = HashMap::, usize>::with_capacity(items.len()); + let mut entries = Vec::::with_capacity(items.len()); -fn stream_epg_reference_map(items: &[StreamEpgItemRequest]) -> HashMap<&str, i64> { - let mut references = HashMap::with_capacity(items.len()); for item in items { - if let Some(reference_ts) = item.reference_ts { - references.entry(item.epg_channel_id.as_str()).or_insert(reference_ts); + let storage_key = canonicalize_untrusted_epg_id(&item.epg_channel_id, output_case); + if let Some(entry_index) = entry_index_by_storage_key.get(storage_key.as_ref()).copied() { + if entries[entry_index].reference_ts.is_none() { + entries[entry_index].reference_ts = item.reference_ts; + } + } else { + entry_index_by_storage_key.insert(Arc::clone(&storage_key), entries.len()); + entries.push(PreparedStreamEpgEntry { storage_key, reference_ts: item.reference_ts }); } } - references + + PreparedStreamEpgRequest { entries } } async fn serve_stream_epg( @@ -532,46 +572,35 @@ async fn serve_stream_epg( user: &ProxyUserCredentials, target: &Arc, epg_path: &Path, - items: Vec, + prepared: PreparedStreamEpgRequest, ) -> StreamEpgResponse { let epg_processing_options = get_epg_processing_options(app_state, user, target); let live_now = chrono::Utc::now().timestamp(); - // Deduplicate by first occurrence, preserving order - let unique_ids: Vec<&str> = { - let mut seen = HashSet::new(); - items - .iter() - .filter_map(|item| { - let epg_channel_id = item.epg_channel_id.as_str(); - seen.insert(epg_channel_id).then_some(epg_channel_id) - }) - .collect() - }; - let query_ids = unique_ids + let PreparedStreamEpgRequest { entries: prepared_entries } = prepared; + let storage_keys = prepared_entries .iter() - .map(|id| (*id).intern()) - .collect::>>(); + .map(|entry| Arc::clone(&entry.storage_key)) + .collect(); - let channels = match epg_query_channels( + let channels: Vec> = match epg_query_channels_by_storage_key( &app_state.app_config.file_locks, epg_path, - query_ids, + storage_keys, ) .await { - Ok(results) => results, + Ok(results) => results.into_iter().map(|(_, channel)| channel).collect(), Err(err) => { error!("{err}"); - unique_ids.into_iter().map(|id| (id.intern(), None)).collect() + std::iter::repeat_with(|| None).take(prepared_entries.len()).collect() } }; - let reference_by_channel = stream_epg_reference_map(&items); let mut entries = Vec::with_capacity(channels.len()); - for (epg_channel_id, channel) in channels { - let reference_ts = reference_by_channel.get(epg_channel_id.as_ref()).copied().unwrap_or(live_now); + for (channel, prepared_entry) in channels.into_iter().zip(prepared_entries) { + let reference_ts = prepared_entry.reference_ts.unwrap_or(live_now); let window_start = reference_ts.saturating_sub(STREAM_EPG_ARCHIVE_WINDOW_BACK_SECS); let window_end = reference_ts.saturating_add(STREAM_EPG_ARCHIVE_WINDOW_FWD_SECS); let programmes = match channel { @@ -580,7 +609,7 @@ async fn serve_stream_epg( }; entries.push(StreamEpgEntry { - epg_channel_id: epg_channel_id.to_string(), + epg_channel_id: prepared_entry.storage_key.to_string(), target_id: Some(target.id), programmes, }); @@ -617,15 +646,26 @@ fn stream_epg_programmes_for_channel( fn group_stream_epg_items( items: Vec, -) -> Result)>, &'static str> { +) -> Result)>, String> { if items.is_empty() { - return Err("items must not be empty"); + return Err("items must not be empty".to_string()); + } + if items.len() > MAX_STREAM_EPG_ITEMS { + return Err(format!("items must not exceed {MAX_STREAM_EPG_ITEMS} entries")); } let mut groups = Vec::<(u16, Vec)>::new(); for item in items { + if item.epg_channel_id.is_empty() { + return Err("epg_channel_id must not be empty".to_string()); + } + if item.epg_channel_id.len() > MAX_STREAM_EPG_CHANNEL_ID_BYTES { + return Err(format!( + "epg_channel_id must not exceed {MAX_STREAM_EPG_CHANNEL_ID_BYTES} bytes" + )); + } let Some(target_id) = item.target_id else { - return Err("all items must include target_id"); + return Err("all items must include target_id".to_string()); }; if let Some((_, group_items)) = groups.iter_mut().find(|(group_target_id, _)| *group_target_id == target_id) { @@ -638,16 +678,12 @@ fn group_stream_epg_items( Ok(groups) } -fn empty_stream_epg_entries(target_id: u16, items: &[StreamEpgItemRequest]) -> Vec { - let mut seen = HashSet::new(); - items +fn empty_stream_epg_entries(target_id: u16, prepared: &PreparedStreamEpgRequest) -> Vec { + prepared + .entries .iter() - .filter_map(|item| { - let epg_channel_id = item.epg_channel_id.as_str(); - seen.insert(epg_channel_id).then_some(epg_channel_id) - }) - .map(|epg_channel_id| StreamEpgEntry { - epg_channel_id: epg_channel_id.to_string(), + .map(|entry| StreamEpgEntry { + epg_channel_id: entry.storage_key.to_string(), target_id: Some(target_id), programmes: Vec::new(), }) @@ -665,7 +701,7 @@ pub(crate) async fn stream_epg_api( ) -> impl IntoResponse + Send { let grouped_items = match group_stream_epg_items(stream_epg_req.items) { Ok(groups) => groups, - Err(message) => return stream_epg_bad_request(message), + Err(message) => return stream_epg_bad_request(&message), }; let config = app_state.app_config.config.load_full(); @@ -676,13 +712,18 @@ pub(crate) async fn stream_epg_api( let Some(target) = app_state.app_config.get_target_by_id(target_id) else { return stream_epg_bad_request(&format!("unknown target_id: {target_id}")); }; + let output_case = EpgIdOutputCase::from_lowercase(target + .options + .as_ref() + .is_some_and(ConfigTargetOptions::lowercase_epg_ids)); + let prepared = prepare_stream_epg_request(&items, output_case); let Some(epg_path) = get_epg_path_for_target(config.as_ref(), &target) else { - entries.extend(empty_stream_epg_entries(target_id, &items)); + entries.extend(empty_stream_epg_entries(target_id, &prepared)); continue; }; - let result = serve_stream_epg(&app_state, &user, &target, &epg_path, items).await; + let result = serve_stream_epg(&app_state, &user, &target, &epg_path, prepared).await; entries.extend(result.entries); } @@ -790,30 +831,43 @@ pub fn xmltv_api_register() -> axum::Router> { #[cfg(test)] mod tests { use super::{ + empty_stream_epg_entries, + from_programme, get_epg_path_for_target, get_epg_path_for_target_by_type, group_stream_epg_items, + prepare_stream_epg_request, rewrite_epg_channel_resource_url, serve_epg, + serve_short_epg, + serve_stream_epg, + stream_epg_api, stream_epg_programmes_for_channel, - stream_epg_reference_map, + MAX_STREAM_EPG_CHANNEL_ID_BYTES, + MAX_STREAM_EPG_ITEMS, }; - use crate::api::model::create_test_app_state; - use crate::model::{ - Config, ConfigTarget, Epg, IcsEpgSourceConfig, M3uTargetOutput, ProxyUserCredentials, TargetOutput, - XtreamTargetFlagsSet, XtreamTargetOutput, + use crate::{ + api::model::{create_test_app_state, AppState}, + model::{ + Config, ConfigTarget, Epg, IcsEpgSourceConfig, M3uTargetOutput, ProxyUserCredentials, + TargetOutput, XtreamTargetFlagsSet, XtreamTargetOutput, + }, + processing::parser::ics::parse_ics_file_to_channel, + repository::{epg_write_file, BPlusTree}, + utils::{lowercase_xmltv_text, EpgIdOutputCase, EpgProcessingOptions, EpgTimeShift}, }; - use crate::processing::parser::ics::parse_ics_file_to_channel; - use crate::repository::epg_write_file; - use crate::utils::{EpgProcessingOptions, EpgTimeShift}; use arc_swap::ArcSwapOption; + use axum::response::IntoResponse; + use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; use shared::{ foundation::Filter, - model::{EpgChannel, EpgProgramme, ProcessingOrder, StreamEpgItemRequest, TargetType}, + model::{ + ConfigTargetOptions, EpgChannel, EpgOutputOptions, EpgProgramme, ProcessingOrder, StreamEpgItemRequest, + StreamEpgRequest, TargetType, + }, utils::{concat_path, obfuscate_text, Internable}, }; - use std::{collections::HashMap, fs}; - use std::sync::Arc; + use std::{collections::HashMap, fs, sync::Arc}; use tempfile::tempdir; fn test_target_with_xtream_and_m3u() -> ConfigTarget { @@ -877,6 +931,48 @@ mod tests { } } + fn test_app_state() -> Arc { + create_test_app_state(Config::default()) + } + + fn test_target_with_epg_options(lowercase_ids: bool, lowercase_display_names: bool) -> Arc { + let mut target = test_target_with_xtream_only(); + target.options = Some(ConfigTargetOptions { + epg_output: EpgOutputOptions { + lowercase_ids, + lowercase_xmltv_display_names: lowercase_display_names, + }, + ..ConfigTargetOptions::default() + }); + Arc::new(target) + } + + fn write_test_epg_db(path: &std::path::Path, channel: EpgChannel) { + write_test_epg_channels(path, [channel]); + } + + fn write_test_epg_channels( + path: &std::path::Path, + channels: impl IntoIterator, + ) { + let mut tree = BPlusTree::, EpgChannel>::new(); + for channel in channels { + tree.insert(Arc::clone(&channel.id), channel); + } + tree.store(path).expect("test EPG DB should be written"); + } + + fn stream_epg_item(id: impl Into, target_id: Option, reference_ts: Option) -> StreamEpgItemRequest { + StreamEpgItemRequest { epg_channel_id: id.into(), target_id, reference_ts } + } + + async fn response_body_text(response: axum::response::Response) -> String { + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .expect("response body should be readable"); + String::from_utf8(body.to_vec()).expect("response body should be UTF-8") + } + #[tokio::test] async fn regular_xmltv_output_contains_imported_ics_channel_and_programme() { let dir = tempdir().expect("temp dir"); @@ -917,6 +1013,7 @@ mod tests { }, &epg_path, &HashMap::, Arc>::new(), + &EpgOutputOptions::default(), ) .expect("write EPG database"); @@ -1014,33 +1111,80 @@ mod tests { #[test] fn test_group_stream_epg_items_rejects_missing_target() { - let err = group_stream_epg_items(vec![StreamEpgItemRequest { - epg_channel_id: "epg-1".to_string(), - target_id: None, - reference_ts: None, - }]) - .unwrap_err(); + let err = group_stream_epg_items(vec![stream_epg_item("epg-1", None, None)]).unwrap_err(); assert_eq!(err, "all items must include target_id"); } + #[test] + fn test_group_stream_epg_items_rejects_empty_channel_id() { + let err = group_stream_epg_items(vec![stream_epg_item("", Some(1), None)]).unwrap_err(); + + assert_eq!(err, "epg_channel_id must not be empty"); + } + + #[test] + fn test_group_stream_epg_items_rejects_overlong_channel_id() { + let err = group_stream_epg_items(vec![stream_epg_item( + "x".repeat(MAX_STREAM_EPG_CHANNEL_ID_BYTES + 1), + Some(1), + None, + )]) + .unwrap_err(); + + assert_eq!( + err, + format!("epg_channel_id must not exceed {MAX_STREAM_EPG_CHANNEL_ID_BYTES} bytes") + ); + } + + #[test] + fn test_group_stream_epg_items_rejects_excessive_item_count() { + let items = (0..=MAX_STREAM_EPG_ITEMS) + .map(|index| stream_epg_item(format!("epg-{index}"), Some(1), None)) + .collect(); + + let err = group_stream_epg_items(items).unwrap_err(); + + assert_eq!(err, format!("items must not exceed {MAX_STREAM_EPG_ITEMS} entries")); + } + + #[test] + fn test_group_stream_epg_items_accepts_configured_limits() { + let mut items = (0..MAX_STREAM_EPG_ITEMS) + .map(|index| stream_epg_item(format!("epg-{index}"), Some(1), None)) + .collect::>(); + items[0].epg_channel_id = "x".repeat(MAX_STREAM_EPG_CHANNEL_ID_BYTES); + + let groups = group_stream_epg_items(items).expect("request at the configured limits should be accepted"); + + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].1.len(), MAX_STREAM_EPG_ITEMS); + } + + #[tokio::test] + async fn stream_epg_api_returns_bad_request_for_invalid_item() { + let response = stream_epg_api( + axum::extract::State(test_app_state()), + axum::extract::Json(StreamEpgRequest { + items: vec![stream_epg_item("", Some(1), None)], + }), + ) + .await + .into_response(); + + assert_eq!(response.status(), axum::http::StatusCode::BAD_REQUEST); + let body = response_body_text(response).await; + let error = serde_json::from_str::(&body) + .expect("bad-request response should contain JSON"); + assert_eq!(error["error"], "epg_channel_id must not be empty"); + } + #[test] fn test_group_stream_epg_items_groups_by_target_in_input_order() { let groups = group_stream_epg_items(vec![ - StreamEpgItemRequest { - epg_channel_id: "epg-1".to_string(), - target_id: Some(2), - reference_ts: None, - }, - StreamEpgItemRequest { - epg_channel_id: "epg-2".to_string(), - target_id: Some(5), - reference_ts: None, - }, - StreamEpgItemRequest { - epg_channel_id: "epg-3".to_string(), - target_id: Some(2), - reference_ts: None, - }, + stream_epg_item("epg-1", Some(2), None), + stream_epg_item("epg-2", Some(5), None), + stream_epg_item("epg-3", Some(2), None), ]) .unwrap(); @@ -1064,28 +1208,524 @@ mod tests { } #[test] - fn stream_epg_reference_map_keeps_first_reference_per_channel() { + fn stream_epg_request_keeps_first_available_reference_per_exact_id() { let items = [ - StreamEpgItemRequest { - epg_channel_id: "epg-1".to_string(), - target_id: Some(1), - reference_ts: Some(100), - }, - StreamEpgItemRequest { - epg_channel_id: "epg-1".to_string(), - target_id: Some(1), - reference_ts: Some(200), - }, - StreamEpgItemRequest { - epg_channel_id: "epg-2".to_string(), - target_id: Some(1), - reference_ts: None, - }, + stream_epg_item("epg-1", Some(1), Some(100)), + stream_epg_item("epg-1", Some(1), Some(200)), + stream_epg_item("epg-2", Some(1), None), + stream_epg_item("epg-2", Some(1), Some(300)), ]; - let references = stream_epg_reference_map(&items); + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::Preserve); - assert_eq!(references.get("epg-1"), Some(&100)); - assert_eq!(references.get("epg-2"), None); + assert_eq!(prepared.entries.len(), 2); + assert_eq!(prepared.entries[0].storage_key.as_ref(), "epg-1"); + assert_eq!(prepared.entries[0].reference_ts, Some(100)); + assert_eq!(prepared.entries[1].storage_key.as_ref(), "epg-2"); + assert_eq!(prepared.entries[1].reference_ts, Some(300)); + } + + #[test] + fn stream_epg_request_canonicalizes_before_deduplication() { + let items = [ + stream_epg_item("Example.Channel", Some(1), Some(100)), + stream_epg_item("example.CHANNEL", Some(1), Some(200)), + stream_epg_item("ÄBC.Id", Some(1), None), + ]; + + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); + let storage_keys = prepared + .entries + .iter() + .map(|entry| entry.storage_key.as_ref()) + .collect::>(); + + assert_eq!(storage_keys, vec!["example.channel", "Äbc.id"]); + assert_eq!(prepared.entries[0].reference_ts, Some(100)); + assert_eq!(prepared.entries[1].reference_ts, None); + } + + #[test] + fn stream_epg_request_uses_non_interned_lookup_ids() { + let globally_interned = "request.example".intern(); + let items = [stream_epg_item("REQUEST.Example", Some(1), None)]; + + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); + + assert_eq!(prepared.entries[0].storage_key.as_ref(), globally_interned.as_ref()); + assert!(!Arc::ptr_eq(&prepared.entries[0].storage_key, &globally_interned)); + + let preserve_prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::Preserve); + let preserve_interned = "REQUEST.Example".intern(); + assert_eq!(preserve_prepared.entries[0].storage_key.as_ref(), preserve_interned.as_ref()); + assert!(!Arc::ptr_eq(&preserve_prepared.entries[0].storage_key, &preserve_interned)); + } + + #[test] + fn stream_epg_request_preserves_case_distinct_entries_by_default() { + let items = [ + stream_epg_item("Example.Channel", Some(1), Some(100)), + stream_epg_item("example.CHANNEL", Some(1), Some(200)), + ]; + + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::Preserve); + + assert_eq!(prepared.entries.len(), 2); + assert_eq!(prepared.entries[0].storage_key.as_ref(), "Example.Channel"); + assert_eq!(prepared.entries[0].reference_ts, Some(100)); + assert_eq!(prepared.entries[1].storage_key.as_ref(), "example.CHANNEL"); + assert_eq!(prepared.entries[1].reference_ts, Some(200)); + } + + #[test] + fn empty_stream_epg_entries_use_prepared_canonical_ids() { + let items = [ + stream_epg_item("Example.Channel", Some(7), None), + stream_epg_item("example.CHANNEL", Some(7), None), + ]; + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); + + let entries = empty_stream_epg_entries(7, &prepared); + + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].epg_channel_id, "example.channel"); + assert_eq!(entries[0].target_id, Some(7)); + assert!(entries[0].programmes.is_empty()); + + let preserve_prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::Preserve); + let preserve_entries = empty_stream_epg_entries(7, &preserve_prepared); + assert_eq!(preserve_entries.len(), 2); + assert_eq!(preserve_entries[0].epg_channel_id, "Example.Channel"); + assert_eq!(preserve_entries[1].epg_channel_id, "example.CHANNEL"); + } + + #[test] + fn short_epg_response_uses_canonical_id_for_both_id_fields() { + let stream_id = "42".intern(); + let epg_id = "example.channel".intern(); + let programme = EpgProgramme::new_all( + 100, + 200, + Arc::clone(&epg_id), + Some("News".intern()), + None, + None, + ); + let options = EpgProcessingOptions { + rewrite_urls: false, + time_shift: EpgTimeShift::None, + encrypt_secret: [0; 16], + }; + + let result = from_programme(&stream_id, &epg_id, &programme, &options); + + assert_eq!(result.epg_id.as_ref(), "example.channel"); + assert_eq!(result.channel_id.as_ref(), "example.channel"); + } + + #[test] + fn lowercased_xmltv_display_name_remains_xml_escaped() { + let display_name = lowercase_xmltv_text("CAFÉ NETWORK & ", true); + let mut writer = quick_xml::Writer::new(Vec::new()); + writer + .write_event(Event::Start(BytesStart::new("display-name"))) + .expect("display-name start should serialize"); + writer + .write_event(Event::Text(BytesText::new(display_name.as_ref()))) + .expect("display-name text should serialize"); + writer + .write_event(Event::End(BytesEnd::new("display-name"))) + .expect("display-name end should serialize"); + + let xml = String::from_utf8(writer.into_inner()).expect("serialized XML should be UTF-8"); + + assert_eq!(xml, "café network & <hd>"); + } + + #[tokio::test] + async fn xmltv_output_uses_canonical_channel_ids_and_only_lowercases_display_names() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("epg.db"); + write_test_epg_db( + &epg_path, + EpgChannel { + id: "example.channel".intern(), + title: Some("CAFÉ NETWORK & ".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + 100, + 200, + "source.MixedCase".intern(), + Some("News & Updates".intern()), + Some("Keep ".intern()), + None, + )], + }, + ); + let app_state = test_app_state(); + let target = test_target_with_epg_options(true, true); + + let response = serve_epg( + &app_state, + &epg_path, + &ProxyUserCredentials::default(), + &target, + None, + ) + .await; + let xml = response_body_text(response).await; + + assert!(xml.contains(r#""#), "XMLTV output: {xml}"); + assert!(xml.contains(r#""#)); + assert!(xml.contains("café network & <hd>")); + assert!(xml.contains("News & Updates")); + assert!(xml.contains("Keep <Case>")); + assert!(!xml.contains("news & updates")); + } + + #[tokio::test] + async fn xmltv_output_preserves_ids_and_display_names_when_options_are_disabled() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("epg.db"); + write_test_epg_db( + &epg_path, + EpgChannel { + id: "Example.Channel".intern(), + title: Some("MixedCase Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new(100, 200, "different.source.case".intern())], + }, + ); + let app_state = test_app_state(); + let target = Arc::new(test_target_with_xtream_only()); + + let response = serve_epg( + &app_state, + &epg_path, + &ProxyUserCredentials::default(), + &target, + None, + ) + .await; + let xml = response_body_text(response).await; + + assert!(xml.contains(r#""#)); + assert!(xml.contains(r#"channel="Example.Channel""#)); + assert!(xml.contains("MixedCase Network")); + } + + #[tokio::test] + async fn xmltv_output_preserves_legacy_mixed_case_key_order_when_options_are_disabled() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("epg.db"); + write_test_epg_channels( + &epg_path, + [ + EpgChannel { + id: "Z.Channel".intern(), + title: Some("First Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new(100, 200, "Z.Channel".intern())], + }, + EpgChannel { + id: "a.channel".intern(), + title: Some("Second Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new(100, 200, "a.channel".intern())], + }, + ], + ); + let app_state = test_app_state(); + let target = Arc::new(test_target_with_xtream_only()); + + let response = serve_epg( + &app_state, + &epg_path, + &ProxyUserCredentials::default(), + &target, + None, + ) + .await; + let xml = response_body_text(response).await; + let first_position = xml.find(r#""#).expect("first channel should exist"); + let second_position = xml.find(r#""#).expect("second channel should exist"); + + assert!(first_position < second_position, "XMLTV output should preserve legacy key order: {xml}"); + } + + #[tokio::test] + async fn xmltv_output_lowercases_only_display_name_when_id_option_is_disabled() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("epg.db"); + write_test_epg_db( + &epg_path, + EpgChannel { + id: "Example.Channel".intern(), + title: Some("CAFÉ NETWORK".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + 100, + 200, + "Example.Channel".intern(), + Some("MixedCase Programme".intern()), + Some("Keep ".intern()), + None, + )], + }, + ); + let app_state = test_app_state(); + let target = test_target_with_epg_options(false, true); + + let response = serve_epg( + &app_state, + &epg_path, + &ProxyUserCredentials::default(), + &target, + None, + ) + .await; + let xml = response_body_text(response).await; + + assert!(xml.contains(r#""#)); + assert!(xml.contains(r#"channel="Example.Channel""#)); + assert!(xml.contains("café network")); + assert!(xml.contains("MixedCase Programme")); + assert!(xml.contains("Keep <Case>")); + } + + #[tokio::test] + async fn xmltv_output_lowercases_only_ids_when_display_name_option_is_disabled() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("epg.db"); + write_test_epg_db( + &epg_path, + EpgChannel { + id: "example.channel".intern(), + title: Some("MixedCase Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + 100, + 200, + "example.channel".intern(), + Some("MixedCase Programme".intern()), + Some("Keep ".intern()), + None, + )], + }, + ); + let app_state = test_app_state(); + let target = test_target_with_epg_options(true, false); + + let response = serve_epg( + &app_state, + &epg_path, + &ProxyUserCredentials::default(), + &target, + None, + ) + .await; + let xml = response_body_text(response).await; + + assert!(xml.contains(r#""#)); + assert!(xml.contains(r#"channel="example.channel""#)); + assert!(xml.contains("MixedCase Network")); + assert!(xml.contains("MixedCase Programme")); + assert!(xml.contains("Keep <Case>")); + } + + #[tokio::test] + async fn short_epg_uses_target_output_case_for_legacy_and_lowercase_storage_keys() { + let dir = tempdir().expect("temp dir"); + let lowercase_epg_path = dir.path().join("lowercase-epg.db"); + let legacy_epg_path = dir.path().join("legacy-epg.db"); + let now = chrono::Utc::now().timestamp(); + write_test_epg_db( + &lowercase_epg_path, + EpgChannel { + id: "example.channel".intern(), + title: Some("Sample Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + now - 60, + now + 3600, + "example.channel".intern(), + Some("Current".intern()), + None, + None, + )], + }, + ); + write_test_epg_db( + &legacy_epg_path, + EpgChannel { + id: "Example.Channel".intern(), + title: Some("Legacy Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + now - 60, + now + 3600, + "Example.Channel".intern(), + Some("Legacy Current".intern()), + None, + None, + )], + }, + ); + let legacy_db_before = fs::read(&legacy_epg_path).expect("legacy EPG DB should be readable"); + let app_state = test_app_state(); + let target = test_target_with_epg_options(true, false); + let request_id = "Example.Channel".intern(); + + let response = serve_short_epg( + &app_state, + &lowercase_epg_path, + &ProxyUserCredentials::default(), + &target, + &request_id, + "42".intern(), + 4, + ) + .await; + let body = response_body_text(response).await; + let result = serde_json::from_str::(&body) + .expect("short EPG response should deserialize"); + + assert_eq!(result.epg_listings.len(), 1); + assert_eq!(result.epg_listings[0].epg_id.as_ref(), "example.channel"); + assert_eq!(result.epg_listings[0].channel_id.as_ref(), "example.channel"); + + let disabled_target = Arc::new(test_target_with_xtream_only()); + let disabled_response = serve_short_epg( + &app_state, + &legacy_epg_path, + &ProxyUserCredentials::default(), + &disabled_target, + &request_id, + "42".intern(), + 4, + ) + .await; + let disabled_body = response_body_text(disabled_response).await; + let disabled_result = serde_json::from_str::(&disabled_body) + .expect("disabled short EPG response should deserialize"); + assert_eq!(disabled_result.epg_listings.len(), 1); + assert_eq!(disabled_result.epg_listings[0].epg_id.as_ref(), "Example.Channel"); + assert_eq!(disabled_result.epg_listings[0].channel_id.as_ref(), "Example.Channel"); + assert_eq!( + fs::read(&legacy_epg_path).expect("legacy EPG DB should remain readable"), + legacy_db_before, + "serving a legacy EPG DB must not rewrite its key schema" + ); + } + + #[tokio::test] + async fn stream_epg_lowercase_mode_deduplicates_case_variants_and_keeps_first_reference() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("epg.db"); + let first_reference = 1_700_000_000; + write_test_epg_db( + &epg_path, + EpgChannel { + id: "example.channel".intern(), + title: Some("Sample Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + first_reference - 60, + first_reference + 60, + "example.channel".intern(), + Some("First Window".intern()), + None, + None, + )], + }, + ); + let app_state = test_app_state(); + let target = test_target_with_epg_options(true, false); + let items = [ + stream_epg_item("Example.Channel", Some(target.id), Some(first_reference)), + stream_epg_item( + "example.CHANNEL", + Some(target.id), + Some(first_reference + 7 * 24 * 3600), + ), + ]; + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::LowercaseAscii); + + let response = serve_stream_epg( + &app_state, + &ProxyUserCredentials::default(), + &target, + &epg_path, + prepared, + ) + .await; + + assert_eq!(response.entries.len(), 1); + assert_eq!(response.entries[0].epg_channel_id, "example.channel"); + assert_eq!(response.entries[0].programmes.len(), 1); + assert_eq!(response.entries[0].programmes[0].title, "First Window"); + } + + #[tokio::test] + async fn stream_epg_preserve_mode_keeps_case_distinct_legacy_entries() { + let dir = tempdir().expect("temp dir"); + let epg_path = dir.path().join("legacy-epg.db"); + let first_reference = 1_700_000_000; + let second_reference = first_reference + 7 * 24 * 3600; + write_test_epg_channels( + &epg_path, + [ + EpgChannel { + id: "Example.Channel".intern(), + title: Some("First Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + first_reference - 60, + first_reference + 60, + "Example.Channel".intern(), + Some("First Window".intern()), + None, + None, + )], + }, + EpgChannel { + id: "example.CHANNEL".intern(), + title: Some("Second Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new_all( + second_reference - 60, + second_reference + 60, + "example.CHANNEL".intern(), + Some("Second Window".intern()), + None, + None, + )], + }, + ], + ); + let app_state = test_app_state(); + let target = Arc::new(test_target_with_xtream_only()); + let items = [ + stream_epg_item("Example.Channel", Some(target.id), Some(first_reference)), + stream_epg_item("example.CHANNEL", Some(target.id), Some(second_reference)), + ]; + let prepared = prepare_stream_epg_request(&items, EpgIdOutputCase::Preserve); + + let response = serve_stream_epg( + &app_state, + &ProxyUserCredentials::default(), + &target, + &epg_path, + prepared, + ) + .await; + + assert_eq!(response.entries.len(), 2); + assert_eq!(response.entries[0].epg_channel_id, "Example.Channel"); + assert_eq!(response.entries[0].programmes.len(), 1); + assert_eq!(response.entries[0].programmes[0].title, "First Window"); + assert_eq!(response.entries[1].epg_channel_id, "example.CHANNEL"); + assert_eq!(response.entries[1].programmes.len(), 1); + assert_eq!(response.entries[1].programmes[0].title, "Second Window"); } fn sample_channel(icon: Option<&str>) -> EpgChannel { diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index f9d44dd6a..234351e52 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1836,6 +1836,7 @@ mod tests { }, &epg_path, &std::collections::HashMap::, Arc>::new(), + &shared::model::EpgOutputOptions::default(), ) .expect("write target EPG"); diff --git a/backend/src/processing/parser/xmltv.rs b/backend/src/processing/parser/xmltv.rs index 813ced7c0..d77b0f63a 100644 --- a/backend/src/processing/parser/xmltv.rs +++ b/backend/src/processing/parser/xmltv.rs @@ -6,9 +6,12 @@ use crate::{ }, processing::{ parser::ics, - processor::{with_folded_epg_id, EpgIdCache}, + processor::EpgIdCache, + }, + utils::{ + async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, parse_xmltv_time, + with_folded_epg_id, }, - utils::{async_file_reader, compressed_file_reader_async::CompressedFileReaderAsync, parse_xmltv_time}, }; use log::error; use quick_xml::events::{BytesStart, BytesText, Event}; diff --git a/backend/src/processing/processor/epg.rs b/backend/src/processing/processor/epg.rs index aa2992068..492a92aa9 100644 --- a/backend/src/processing/processor/epg.rs +++ b/backend/src/processing/processor/epg.rs @@ -2,6 +2,7 @@ use crate::model::Epg; use crate::model::{EpgConfig, EpgSmartMatchConfig}; use crate::model::FetchedPlaylist; use crate::processing::parser::xmltv::normalize_channel_name; +use crate::utils::with_folded_epg_id; use log::{debug, trace, warn}; use rphonetic::{DoubleMetaphone, Encoder}; use std::collections::{HashMap, HashSet}; @@ -9,30 +10,6 @@ use shared::model::{EpgSmartMatchConfigDto, PlaylistItem, XtreamCluster}; use std::sync::Arc; use shared::utils::Internable; -const EPG_ID_STACK_FOLD_LEN: usize = 128; - -/// Runs `f` with an ASCII-lowercased EPG id. -/// -/// Non-ASCII characters are left unchanged, matching `eq_ignore_ascii_case` semantics. -pub(crate) fn with_folded_epg_id(id: &str, f: impl FnOnce(&str) -> R) -> R { - if !id.bytes().any(|byte| byte.is_ascii_uppercase()) { - return f(id); - } - - if id.len() <= EPG_ID_STACK_FOLD_LEN { - let mut buf = [0_u8; EPG_ID_STACK_FOLD_LEN]; - for (out, byte) in buf.iter_mut().zip(id.bytes()) { - *out = byte.to_ascii_lowercase(); - } - let folded = std::str::from_utf8(&buf[..id.len()]); - return f(folded.unwrap_or(id)); - } - - let mut folded = String::with_capacity(id.len()); - folded.extend(id.chars().map(|ch| ch.to_ascii_lowercase())); - f(&folded) -} - pub struct EpgIdCache { /// Membership set of the epg ids collected from playlist channels. /// Keys are stored ASCII-lowercased so guide-id matching is case-insensitive. diff --git a/backend/src/repository/epg_repository.rs b/backend/src/repository/epg_repository.rs index 61ebaf7e8..e9ce9c608 100644 --- a/backend/src/repository/epg_repository.rs +++ b/backend/src/repository/epg_repository.rs @@ -2,9 +2,11 @@ use crate::model::{Config, ConfigTarget, TargetOutput}; use crate::model::{Epg}; use crate::repository::{m3u_get_epg_file_path_for_target, BPlusTree}; use crate::repository::{xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTreeQuery}; -use crate::utils::{debug_if_enabled, FileLockManager}; +use crate::utils::{ + canonicalize_output_epg_id, debug_if_enabled, EpgIdOutputCase, FileLockManager, +}; use shared::error::{ TuliproxError}; -use shared::model::{EpgChannel, PlaylistGroup}; +use shared::model::{EpgChannel, EpgOutputOptions, PlaylistGroup}; use std::collections::HashMap; use std::path::Path; use std::sync::Arc; @@ -29,21 +31,28 @@ pub fn epg_write_file( epg: &Epg, path: &Path, rename_map: &HashMap, Arc, S>, + epg_output: &EpgOutputOptions, ) -> Result<(), TuliproxError> { if epg.children.is_empty() { return Ok(()); } let mut tree = BPlusTree::, EpgChannel>::new(); + let output_case = EpgIdOutputCase::from_lowercase(epg_output.lowercase_ids); for channel in &epg.children { - if !channel.programmes.is_empty() { - let mut chan = (**channel).clone(); - if let Some(title) = rename_map.get(&chan.id) { - chan.title = Some(Arc::clone(title)); - } - chan.programmes.sort_by_key(|p| p.start); - tree.insert(Arc::clone(&channel.id), chan); + if channel.programmes.is_empty() { + continue; } + + let mut chan = (**channel).clone(); + let output_id = canonicalize_output_epg_id(&chan.id, output_case); + if let Some(title) = rename_map.get(&output_id) { + chan.title = Some(Arc::clone(title)); + } + + chan.id = Arc::clone(&output_id); + chan.programmes.sort_by_key(|programme| programme.start); + tree.insert(output_id, chan); } tree.store(path).map_err(|err| TuliproxError::RepositoryEpg(format!("Failed to write epg for target {}: {} - {err}", target_name, path.display())))?; @@ -52,14 +61,29 @@ pub fn epg_write_file( Ok(()) } -fn build_epg_rename_map(playlist: Option<&[PlaylistGroup]>) -> HashMap, Arc> { +fn build_epg_rename_map( + playlist: Option<&[PlaylistGroup]>, + output_case: EpgIdOutputCase, +) -> HashMap, Arc> { let mut rename_map = HashMap::new(); if let Some(pl) = playlist { for group in pl { for channel in &group.channels { if let Some(epg_id) = &channel.header.epg_channel_id { if !epg_id.is_empty() { - rename_map.insert(Arc::clone(epg_id), Arc::clone(&channel.header.name)); + let output_id = canonicalize_output_epg_id(epg_id, output_case); + match output_case { + // Preserve the legacy exact-key last-match behavior when the feature is disabled. + EpgIdOutputCase::Preserve => { + rename_map.insert(output_id, Arc::clone(&channel.header.name)); + } + // Canonical collisions use playlist traversal order: the first entry wins. + EpgIdOutputCase::LowercaseAscii => { + rename_map + .entry(output_id) + .or_insert_with(|| Arc::clone(&channel.header.name)); + } + } } } } @@ -77,7 +101,9 @@ pub async fn epg_write_for_target(cfg: &Config, target: &ConfigTarget, target_pa return Ok(()); } if let Some(epg_data) = epg { - let rename_map = Arc::new(build_epg_rename_map(playlist)); + let epg_output = target.options.as_ref().map_or_else(EpgOutputOptions::default, |options| options.epg_output.clone()); + let output_case = EpgIdOutputCase::from_lowercase(epg_output.lowercase_ids); + let rename_map = Arc::new(build_epg_rename_map(playlist, output_case)); let epg_data = Arc::new(epg_data.clone()); match output { TargetOutput::Xtream(_) => { @@ -89,9 +115,10 @@ pub async fn epg_write_for_target(cfg: &Config, target: &ConfigTarget, target_pa let target_name_err = target_name.clone(); let rename_map = Arc::clone(&rename_map); let epg_data = Arc::clone(&epg_data); + let epg_output = epg_output.clone(); let epg_path = epg_path.clone(); tokio::task::spawn_blocking(move || { - epg_write_file(&target_name, &epg_data, &epg_path, &rename_map) + epg_write_file(&target_name, &epg_data, &epg_path, &rename_map, &epg_output) }) .await .map_err(|err| TuliproxError::RepositoryEpg(format!( @@ -113,9 +140,10 @@ pub async fn epg_write_for_target(cfg: &Config, target: &ConfigTarget, target_pa let target_name_err = target_name.clone(); let rename_map = Arc::clone(&rename_map); let epg_data = Arc::clone(&epg_data); + let epg_output = epg_output.clone(); let path = path.clone(); tokio::task::spawn_blocking(move || { - epg_write_file(&target_name, &epg_data, &path, &rename_map) + epg_write_file(&target_name, &epg_data, &path, &rename_map, &epg_output) }) .await .map_err(|err| TuliproxError::RepositoryEpg(format!( @@ -128,12 +156,14 @@ pub async fn epg_write_for_target(cfg: &Config, target: &ConfigTarget, target_pa Ok(()) } -/// Queries EPG channels by their ids. Encapsulates lock acquisition, `BPlusTree` open, and batch lookup. -/// Returns channels in the same order as `query_ids`, with None for ids not found in the DB. -pub async fn epg_query_channels( +/// Queries EPG channels by exact target-storage keys. +/// +/// Callers must prepare each key according to the target's configured output case. Results preserve +/// the order of `storage_keys` and contain `None` for keys that are not present in the database. +pub async fn epg_query_channels_by_storage_key( file_locks: &FileLockManager, epg_path: &Path, - query_ids: Vec>, + storage_keys: Vec>, ) -> Result, Option)>, TuliproxError> { let file_lock = file_locks.read_lock(epg_path).await; let epg_path = epg_path.to_path_buf(); @@ -143,11 +173,11 @@ pub async fn epg_query_channels( let mut query = BPlusTreeQuery::, EpgChannel>::try_new(&epg_path) .map_err(|e| TuliproxError::RepositoryEpg(format!("failed to open epg db {}: {e}", epg_path.display())))?; - let mut results = Vec::with_capacity(query_ids.len()); - for channel_id in &query_ids { - let channel = query.query(channel_id) + let mut results = Vec::with_capacity(storage_keys.len()); + for storage_key in &storage_keys { + let channel = query.query(storage_key) .map_err(|e| TuliproxError::RepositoryEpg(format!("failed to query epg db {}: {e}", epg_path.display())))?; - results.push((Arc::clone(channel_id), channel)); + results.push((Arc::clone(storage_key), channel)); } Ok(results) }) @@ -163,9 +193,13 @@ mod tests { use crate::repository::BPlusTree; use crate::utils::FileLockManager; use arc_swap::ArcSwapOption; - use shared::foundation::Filter; - use shared::model::{EpgChannel, ProcessingOrder}; - use shared::utils::Internable; + use shared::{ + foundation::Filter, + model::{ + EpgChannel, EpgProgramme, PlaylistItem, PlaylistItemHeader, ProcessingOrder, XtreamCluster, + }, + utils::Internable, + }; use tempfile::TempDir; fn target_with_m3u_and_xtream() -> ConfigTarget { @@ -200,7 +234,7 @@ mod tests { } #[tokio::test] - async fn epg_query_channels_returns_found_channels_in_order() { + async fn epg_query_channels_by_storage_key_returns_found_channels_in_order() { let tmp = TempDir::new().expect("temp dir created"); let path = tmp.path().join("epg.db"); @@ -221,10 +255,11 @@ mod tests { tree.store(&path).expect("store epg"); let file_locks = FileLockManager::new(); - let query_ids = vec!["ch1".intern(), "ch2".intern(), "ch3".intern()]; + let storage_keys = vec!["ch1".intern(), "ch2".intern(), "ch3".intern()]; - let results = epg_query_channels(&file_locks, &path, query_ids).await - .expect("epg_query_channels succeeds"); + let results = epg_query_channels_by_storage_key(&file_locks, &path, storage_keys) + .await + .expect("EPG storage-key query succeeds"); assert_eq!(results.len(), 3); assert_eq!(results[0].0.as_ref(), "ch1"); @@ -292,7 +327,7 @@ mod tests { let locks = FileLockManager::new(); for epg_path in [&m3u_path, &xtream_path] { - let results = epg_query_channels(&locks, epg_path, vec!["f1.calendar".intern()]) + let results = epg_query_channels_by_storage_key(&locks, epg_path, vec!["f1.calendar".intern()]) .await .expect("read target EPG"); let stored = results[0].1.as_ref().expect("stored ICS channel"); @@ -301,4 +336,152 @@ mod tests { assert_eq!(stored.programmes[0].title.as_deref(), Some("Formula 1 Practice")); } } + + fn mixed_case_epg() -> Epg { + Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(EpgChannel { + id: "Example.Channel".intern(), + title: Some("Guide Name".intern()), + icon: None, + programmes: vec![ + EpgProgramme::new(20, 30, "Example.Channel".intern()), + EpgProgramme::new(10, 20, "Example.Channel".intern()), + ], + })], + } + } + + fn rename_playlist(channels: &[(&str, &str)]) -> Vec { + vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: channels + .iter() + .map(|(epg_id, name)| PlaylistItem { + header: PlaylistItemHeader { + name: name.intern(), + epg_channel_id: Some(epg_id.intern()), + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }) + .collect(), + xtream_cluster: XtreamCluster::Live, + }] + } + + #[test] + fn epg_write_file_lowercases_visible_id_and_renames_with_canonical_key() { + let tmp = TempDir::new().expect("temp dir created"); + let path = tmp.path().join("epg.db"); + let playlist = rename_playlist(&[("example.CHANNEL", "Mapped Name")]); + let rename_map = build_epg_rename_map(Some(&playlist), EpgIdOutputCase::LowercaseAscii); + let options = EpgOutputOptions { lowercase_ids: true, ..EpgOutputOptions::default() }; + + epg_write_file("target", &mixed_case_epg(), &path, &rename_map, &options) + .expect("canonical EPG should be written"); + + let mut query = BPlusTreeQuery::, EpgChannel>::try_new(&path).expect("EPG DB should open"); + let channel = query + .query(&"example.channel".intern()) + .expect("canonical EPG key should be queryable") + .expect("canonical EPG channel should exist"); + assert_eq!(channel.id.as_ref(), "example.channel"); + assert_eq!(channel.title.as_deref(), Some("Mapped Name")); + assert_eq!(channel.programmes.iter().map(|programme| programme.start).collect::>(), vec![10, 20]); + assert!(query + .query(&"Example.Channel".intern()) + .expect("mixed-case EPG key query should succeed") + .is_none()); + } + + #[test] + fn epg_write_file_preserves_guide_case_and_exact_rename_behavior_when_disabled() { + let tmp = TempDir::new().expect("temp dir created"); + let path = tmp.path().join("epg.db"); + let playlist = rename_playlist(&[("example.channel", "Mapped Name")]); + let rename_map = build_epg_rename_map(Some(&playlist), EpgIdOutputCase::Preserve); + + epg_write_file("target", &mixed_case_epg(), &path, &rename_map, &EpgOutputOptions::default()) + .expect("case-preserving EPG should be written"); + + let mut query = BPlusTreeQuery::, EpgChannel>::try_new(&path).expect("EPG DB should open"); + let channel = query + .query(&"Example.Channel".intern()) + .expect("source-case EPG key should be queryable") + .expect("guide-case EPG channel should exist"); + assert_eq!(channel.id.as_ref(), "Example.Channel"); + assert_eq!(channel.title.as_deref(), Some("Guide Name")); + assert!(query + .query(&"example.channel".intern()) + .expect("lowercase EPG key query should succeed") + .is_none()); + } + + #[test] + fn epg_write_file_keeps_same_case_rename_behavior_when_disabled() { + let tmp = TempDir::new().expect("temp dir created"); + let path = tmp.path().join("epg.db"); + let playlist = rename_playlist(&[("Example.Channel", "Mapped Name")]); + let rename_map = build_epg_rename_map(Some(&playlist), EpgIdOutputCase::Preserve); + + epg_write_file("target", &mixed_case_epg(), &path, &rename_map, &EpgOutputOptions::default()) + .expect("case-preserving EPG should be written"); + + let mut query = BPlusTreeQuery::, EpgChannel>::try_new(&path).expect("EPG DB should open"); + let channel = query + .query(&"Example.Channel".intern()) + .expect("source-case EPG key should be queryable") + .expect("EPG channel should exist"); + assert_eq!(channel.id.as_ref(), "Example.Channel"); + assert_eq!(channel.title.as_deref(), Some("Mapped Name")); + } + + #[test] + fn epg_write_file_preserves_legacy_key_order_when_disabled() { + let tmp = TempDir::new().expect("temp dir created"); + let path = tmp.path().join("epg.db"); + let epg = Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![ + Arc::new(EpgChannel { + id: "Z.Channel".intern(), + title: Some("First Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new(10, 20, "Z.Channel".intern())], + }), + Arc::new(EpgChannel { + id: "a.channel".intern(), + title: Some("Second Network".intern()), + icon: None, + programmes: vec![EpgProgramme::new(10, 20, "a.channel".intern())], + }), + ], + }; + + epg_write_file("target", &epg, &path, &HashMap::new(), &EpgOutputOptions::default()) + .expect("case-preserving EPG should be written"); + + let mut query = BPlusTreeQuery::, EpgChannel>::try_new(&path).expect("EPG DB should open"); + let stored_ids = query.iter().map(|(_, channel)| channel.id).collect::>(); + + assert_eq!(stored_ids.iter().map(AsRef::as_ref).collect::>(), vec!["Z.Channel", "a.channel"]); + } + + #[test] + fn canonical_rename_collision_uses_first_playlist_entry() { + let playlist = rename_playlist(&[ + ("Example.Channel", "First Name"), + ("example.channel", "Second Name"), + ]); + + let rename_map = build_epg_rename_map(Some(&playlist), EpgIdOutputCase::LowercaseAscii); + + assert_eq!(rename_map.get("example.channel").map(AsRef::as_ref), Some("First Name")); + } } diff --git a/backend/src/repository/playlist_repository.rs b/backend/src/repository/playlist_repository.rs index 0effe4ac2..9c112de33 100644 --- a/backend/src/repository/playlist_repository.rs +++ b/backend/src/repository/playlist_repository.rs @@ -15,11 +15,11 @@ use crate::repository::{ MediaServerDiskPlaylistSource, XtreamDiskPlaylistSource, }; use crate::repository::{TargetIdMapping, VirtualIdRecord}; -use crate::utils; +use crate::utils::{self, fold_epg_id_arc}; use log::{debug, info, warn}; use shared::error::{ TuliproxError}; use shared::model::xtream_const::XTREAM_CLUSTER; -use shared::model::{InputPersistence, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, StreamProperties, UUIDType, VirtualId, XtreamCluster, XtreamPlaylistItem}; +use shared::model::{ConfigTargetOptions, InputPersistence, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, StreamProperties, UUIDType, VirtualId, XtreamCluster, XtreamPlaylistItem}; use shared::utils::{is_dash_url, is_hls_url, Internable}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -35,6 +35,28 @@ pub struct ProviderEpisodeKey { pub(crate) virtual_id: u32, } +fn normalize_target_playlist_epg_ids( + playlist: &mut [PlaylistGroup], + target_options: Option<&ConfigTargetOptions>, +) { + if !target_options.is_some_and(ConfigTargetOptions::lowercase_epg_ids) { + return; + } + + for group in playlist { + for channel in &mut group.channels { + let Some(epg_id) = channel.header.epg_channel_id.as_mut() else { + continue; + }; + if epg_id.is_empty() { + continue; + } + + *epg_id = fold_epg_id_arc(epg_id); + } + } +} + #[allow(clippy::too_many_lines)] pub async fn persist_playlist(app_config: &Arc, playlist: &mut [PlaylistGroup], epg: Option<&Epg>, target: &ConfigTarget, playlist_state: Option<&Arc>) -> Result<(), Vec> { @@ -109,6 +131,8 @@ pub async fn persist_playlist(app_config: &Arc, playlist: &mut [Playl drop(provider_series); drop(media_server_series); + normalize_target_playlist_epg_ids(playlist, target.options.as_ref()); + for output in &target.output { let mut filtered: Option> = output.filter().and_then(|flt| apply_filter_to_playlist(playlist, flt)); @@ -754,14 +778,16 @@ mod tests { use super::{ assign_local_series_info_episode_key, assign_media_server_series_info_episode, get_input_media_server_playlist_file_path, materialize_media_server_series_info_episodes, - rewrite_local_series_info_episode_virtual_id, rewrite_series_episode_parent_virtual_ids, - rewrite_series_info_episode_virtual_id, skipped_clusters, LocalEpisodeKey, ProviderEpisodeKey, + normalize_target_playlist_epg_ids, rewrite_local_series_info_episode_virtual_id, + rewrite_series_episode_parent_virtual_ids, rewrite_series_info_episode_virtual_id, skipped_clusters, + LocalEpisodeKey, ProviderEpisodeKey, }; use crate::repository::{BPlusTreeQuery, TargetIdMapping, VirtualIdRecord}; use shared::model::{ - EpisodeStreamProperties, PlaylistEntry, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, - SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, SeriesStreamDetailSeasonProperties, - SeriesStreamProperties, StreamProperties, UUIDType, XtreamCluster, XtreamPlaylistItem, + ConfigTargetOptions, EpgOutputOptions, EpisodeStreamProperties, M3uPlaylistItem, PlaylistEntry, PlaylistGroup, + PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamDetailEpisodeProperties, + SeriesStreamDetailProperties, SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, + UUIDType, XtreamCluster, XtreamPlaylistItem, }; use shared::utils::Internable; use std::{collections::HashMap, sync::Arc}; @@ -783,6 +809,78 @@ mod tests { assert!(skipped.contains(&XtreamCluster::Video)); } + fn epg_normalization_options(enabled: bool) -> ConfigTargetOptions { + ConfigTargetOptions { + epg_output: EpgOutputOptions { lowercase_ids: enabled, ..EpgOutputOptions::default() }, + ..ConfigTargetOptions::default() + } + } + + fn epg_normalization_playlist() -> Vec { + let channel = |epg_channel_id: Option<&str>, name: &str| PlaylistItem { + header: PlaylistItemHeader { + name: name.intern(), + title: "Visible Title".intern(), + group: "Visible Group".intern(), + epg_channel_id: epg_channel_id.map(Internable::intern), + item_type: PlaylistItemType::Live, + xtream_cluster: XtreamCluster::Live, + ..PlaylistItemHeader::default() + }, + }; + + vec![PlaylistGroup { + id: 1, + title: "Live".intern(), + channels: vec![ + channel(Some("Example.Channel"), "Mixed Case"), + channel(Some("already.lower"), "Lowercase"), + channel(Some(""), "Empty"), + channel(None, "Missing"), + ], + xtream_cluster: XtreamCluster::Live, + }] + } + + #[test] + fn target_playlist_epg_normalization_is_consistent_for_m3u_and_xtream() { + let options = epg_normalization_options(true); + let mut playlist = epg_normalization_playlist(); + let lowercase_id = Arc::clone( + playlist[0].channels[1].header.epg_channel_id.as_ref().expect("lowercase EPG ID should exist"), + ); + + normalize_target_playlist_epg_ids(&mut playlist, Some(&options)); + + let mixed = &playlist[0].channels[0]; + assert_eq!(mixed.header.epg_channel_id.as_deref(), Some("example.channel")); + assert_eq!(mixed.header.name.as_ref(), "Mixed Case"); + assert_eq!(mixed.header.title.as_ref(), "Visible Title"); + assert_eq!(mixed.header.group.as_ref(), "Visible Group"); + assert!(Arc::ptr_eq( + playlist[0].channels[1].header.epg_channel_id.as_ref().expect("lowercase EPG ID should remain"), + &lowercase_id, + )); + assert_eq!(playlist[0].channels[2].header.epg_channel_id.as_deref(), Some("")); + assert!(playlist[0].channels[3].header.epg_channel_id.is_none()); + + let m3u = M3uPlaylistItem::from(mixed); + let xtream = XtreamPlaylistItem::from(mixed); + assert_eq!(m3u.epg_channel_id.as_deref(), Some("example.channel")); + assert_eq!(xtream.epg_channel_id.as_deref(), Some("example.channel")); + assert!(m3u.to_m3u(None, false).contains(r#"tvg-id="example.channel""#)); + } + + #[test] + fn target_playlist_epg_normalization_preserves_ids_when_disabled() { + let options = epg_normalization_options(false); + let mut playlist = epg_normalization_playlist(); + + normalize_target_playlist_epg_ids(&mut playlist, Some(&options)); + + assert_eq!(playlist[0].channels[0].header.epg_channel_id.as_deref(), Some("Example.Channel")); + } + fn make_local_series_info(series_uuid: &str, episodes: Vec<(u32, &str, &str)>) -> PlaylistItem { let episode_props = episodes .into_iter() diff --git a/backend/src/utils/epg_id.rs b/backend/src/utils/epg_id.rs new file mode 100644 index 000000000..c0042ee5c --- /dev/null +++ b/backend/src/utils/epg_id.rs @@ -0,0 +1,214 @@ +use shared::utils::Internable; +use std::{borrow::Cow, sync::Arc}; + +const EPG_ID_STACK_FOLD_LEN: usize = 128; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EpgIdOutputCase { + Preserve, + LowercaseAscii, +} + +impl EpgIdOutputCase { + pub(crate) const fn from_lowercase(lowercase: bool) -> Self { + if lowercase { + Self::LowercaseAscii + } else { + Self::Preserve + } + } +} + +/// Runs `f` with an ASCII-lowercase EPG ID. +/// +/// Non-ASCII characters remain unchanged, matching `eq_ignore_ascii_case` semantics. +pub(crate) fn with_folded_epg_id(id: &str, f: impl FnOnce(&str) -> R) -> R { + if !id.bytes().any(|byte| byte.is_ascii_uppercase()) { + return f(id); + } + + if id.len() <= EPG_ID_STACK_FOLD_LEN { + let mut buf = [0_u8; EPG_ID_STACK_FOLD_LEN]; + for (out, byte) in buf.iter_mut().zip(id.bytes()) { + *out = byte.to_ascii_lowercase(); + } + let folded = std::str::from_utf8(&buf[..id.len()]); + return f(folded.unwrap_or(id)); + } + + let mut folded = String::with_capacity(id.len()); + folded.extend(id.chars().map(|ch| ch.to_ascii_lowercase())); + f(&folded) +} + +/// Folds a trusted, persisted EPG ID and interns only a changed value. +pub(crate) fn fold_epg_id_arc(id: &Arc) -> Arc { + with_folded_epg_id(id, |folded| { + if folded == id.as_ref() { + Arc::clone(id) + } else { + folded.intern() + } + }) +} + +/// Folds an untrusted request ID without adding it to the global interner. +pub(crate) fn fold_untrusted_epg_id(id: &str) -> Arc { + let mut folded = id.to_owned(); + folded.make_ascii_lowercase(); + Arc::from(folded) +} + +/// Applies the target output casing to an untrusted request ID without interning it. +pub(crate) fn canonicalize_untrusted_epg_id(id: &str, output_case: EpgIdOutputCase) -> Arc { + match output_case { + EpgIdOutputCase::Preserve => Arc::from(id), + EpgIdOutputCase::LowercaseAscii => fold_untrusted_epg_id(id), + } +} + +/// Applies the configured casing to a visible, persisted EPG ID. +pub(crate) fn canonicalize_output_epg_id(id: &Arc, output_case: EpgIdOutputCase) -> Arc { + match output_case { + EpgIdOutputCase::Preserve => Arc::clone(id), + EpgIdOutputCase::LowercaseAscii => fold_epg_id_arc(id), + } +} + +/// Applies Unicode lowercase only to visible XMLTV text. +pub(crate) fn lowercase_xmltv_text(value: &str, enabled: bool) -> Cow<'_, str> { + if !enabled { + return Cow::Borrowed(value); + } + + let lowercase = value.to_lowercase(); + if lowercase == value { + Cow::Borrowed(value) + } else { + Cow::Owned(lowercase) + } +} + +#[cfg(test)] +mod tests { + use super::{ + canonicalize_output_epg_id, canonicalize_untrusted_epg_id, fold_epg_id_arc, + fold_untrusted_epg_id, lowercase_xmltv_text, EpgIdOutputCase, + }; + use quick_xml::{ + events::{BytesText, Event}, + Writer, + }; + use shared::utils::Internable; + use std::{borrow::Cow, sync::Arc}; + + #[test] + fn fold_epg_id_arc_lowercases_ascii() { + let id = "Example.Channel".intern(); + + let canonical = fold_epg_id_arc(&id); + + assert_eq!(canonical.as_ref(), "example.channel"); + } + + #[test] + fn fold_epg_id_arc_reuses_already_folded_arc() { + let id = "example.channel".intern(); + + let canonical = fold_epg_id_arc(&id); + + assert!(Arc::ptr_eq(&canonical, &id)); + } + + #[test] + fn output_epg_id_reuses_arc_when_case_is_preserved() { + let id = "Example.Channel".intern(); + + let canonical = canonicalize_output_epg_id(&id, EpgIdOutputCase::Preserve); + + assert!(Arc::ptr_eq(&canonical, &id)); + } + + #[test] + fn fold_untrusted_epg_id_handles_long_ids() { + let id = format!("{}.Example", "A".repeat(129)); + + let canonical = fold_untrusted_epg_id(&id); + + assert_eq!(canonical.as_ref(), format!("{}.example", "a".repeat(129))); + } + + #[test] + fn fold_epg_id_arc_handles_ids_longer_than_stack_buffer() { + let id = format!("{}.Example", "A".repeat(129)).intern(); + + let canonical = fold_epg_id_arc(&id); + + assert_eq!(canonical.as_ref(), format!("{}.example", "a".repeat(129))); + } + + #[test] + fn fold_epg_id_arc_preserves_non_ascii_characters() { + let id = "ÄBC.Id".intern(); + + let canonical = fold_epg_id_arc(&id); + + assert_eq!(canonical.as_ref(), "Äbc.id"); + } + + #[test] + fn fold_untrusted_epg_id_does_not_reuse_the_global_interner() { + let interned = "request.example".intern(); + + let request_id = fold_untrusted_epg_id("REQUEST.Example"); + + assert_eq!(request_id.as_ref(), interned.as_ref()); + assert!(!Arc::ptr_eq(&request_id, &interned)); + } + + #[test] + fn untrusted_epg_id_preserves_case_without_using_the_global_interner() { + let interned = "Request.Example".intern(); + + let request_id = canonicalize_untrusted_epg_id("Request.Example", EpgIdOutputCase::Preserve); + + assert_eq!(request_id.as_ref(), interned.as_ref()); + assert!(!Arc::ptr_eq(&request_id, &interned)); + } + + #[test] + fn lowercase_xmltv_text_uses_unicode_lowercase() { + let lowercase = lowercase_xmltv_text("CAFÉ NETWORK", true); + + assert_eq!(lowercase, "café network"); + } + + #[test] + fn lowercase_xmltv_text_handles_unicode_titlecase_characters() { + let lowercase = lowercase_xmltv_text("\u{01C5}", true); + + assert_eq!(lowercase, "\u{01C6}"); + } + + #[test] + fn lowercase_xmltv_text_borrows_unchanged_values() { + let disabled = lowercase_xmltv_text("MixedCase", false); + let already_lowercase = lowercase_xmltv_text("lowercase", true); + + assert!(matches!(disabled, Cow::Borrowed("MixedCase"))); + assert!(matches!(already_lowercase, Cow::Borrowed("lowercase"))); + } + + #[test] + fn lowercase_xmltv_text_remains_xml_escaped() { + let display_name = lowercase_xmltv_text("CAFÉ NETWORK & ", true); + let mut writer = Writer::new(Vec::new()); + + writer + .write_event(Event::Text(BytesText::new(display_name.as_ref()))) + .expect("XMLTV display name should serialize"); + let output = String::from_utf8(writer.into_inner()).expect("XML output should be UTF-8"); + + assert_eq!(output, "café network & <hd>"); + } +} diff --git a/backend/src/utils/mod.rs b/backend/src/utils/mod.rs index 0d40dd381..87c608549 100644 --- a/backend/src/utils/mod.rs +++ b/backend/src/utils/mod.rs @@ -16,6 +16,7 @@ pub(crate) mod geoip; mod db_viewer; pub(crate) mod stream_history_viewer; mod epg_parser; +mod epg_id; mod ordinal; pub mod ffmpeg; mod lru_cache; @@ -70,6 +71,7 @@ pub use with; pub use self::binary_utils::*; pub use self::db_viewer::*; +pub(crate) use self::epg_id::*; pub use self::epg_parser::*; pub use self::geoip::*; pub use self::logging::*; diff --git a/config/source.yml b/config/source.yml index 2812cc5c9..b1cd3d88c 100644 --- a/config/source.yml +++ b/config/source.yml @@ -26,6 +26,9 @@ sources: filter: "!final_channel_lineup!" options: ignore_logo: false + epg_output: + lowercase_ids: false + lowercase_xmltv_display_names: false share_live_streams: hls: true mpeg_ts: true @@ -62,6 +65,9 @@ sources: filter: "!final_channel_lineup!" options: ignore_logo: false + epg_output: + lowercase_ids: false + lowercase_xmltv_display_names: false share_live_streams: hls: true mpeg_ts: true diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index 3babf8bbc..4203dac64 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -1016,6 +1016,9 @@ sources: sort: { } options: ignore_logo: false + epg_output: + lowercase_ids: false + lowercase_xmltv_display_names: false share_live_streams: hls: false mpeg_ts: false @@ -1287,6 +1290,9 @@ targets: use_output: xtream options: ignore_logo: false + epg_output: + lowercase_ids: true + lowercase_xmltv_display_names: false share_live_streams: hls: true mpeg_ts: true @@ -1295,13 +1301,15 @@ targets: #### Target Option Parameters -| Parameter | Type | Required | Default | Technical Impact & Background | -|:-----------------------------|:-----|:--------:|:--------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ignore_logo` | Bool | No | `false` | Ignores `tvg-logo` and `tvg-logo-small` attributes. This reduces downstream device-side logo caching and can keep generated M3U playlists leaner for clients with limited storage or poor cache invalidation behavior. | -| `share_live_streams.hls` | Bool | No | `false` | Enables HLS live sharing for the new HLS cache proxy path. This is a configuration switch for the HLS cache feature and is independent from MPEG-TS stream sharing. | -| `share_live_streams.mpeg_ts` | Bool | No | `false` | Allows Tuliprox to share MPEG-TS live stream connections in reverse proxy mode. This can reduce upstream provider connection usage when multiple clients watch the same channel, but it increases memory usage per shared channel. | -| `remove_duplicates` | Bool | No | `false` | Attempts to remove duplicate entries by `url`. This improves playlist cleanliness and reduces confusing duplicates in the client-facing output. | -| `force_redirect` | Bool | No | `false` | Optional redirect-related behavior switch. This influences how Tuliprox serves final stream delivery where redirect-style output handling is required by the deployment model. | +| Parameter | Type | Required | Default | Technical Impact & Background | +|:-------------------------------------------|:-----|:--------:|:--------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ignore_logo` | Bool | No | `false` | Ignores `tvg-logo` and `tvg-logo-small` attributes. This reduces downstream device-side logo caching and can keep generated M3U playlists leaner for clients with limited storage or poor cache invalidation behavior. | +| `share_live_streams.hls` | Bool | No | `false` | Enables HLS live sharing for the new HLS cache proxy path. This is a configuration switch for the HLS cache feature and is independent from MPEG-TS stream sharing. | +| `share_live_streams.mpeg_ts` | Bool | No | `false` | Allows Tuliprox to share MPEG-TS live stream connections in reverse proxy mode. This can reduce upstream provider connection usage when multiple clients watch the same channel, but it increases memory usage per shared channel. | +| `remove_duplicates` | Bool | No | `false` | Attempts to remove duplicate entries by `url`. This improves playlist cleanliness and reduces confusing duplicates in the client-facing output. | +| `epg_output.lowercase_ids` | Bool | No | `false` | Canonicalizes visible technical EPG IDs with ASCII lowercase across M3U `tvg-id`, Xtream `epg_channel_id`, XMLTV channel/programme references, and EPG API responses. Changing this option requires a full target refresh. | +| `epg_output.lowercase_xmltv_display_names` | Bool | No | `false` | Applies Unicode lowercase exclusively to XMLTV `` values during serialization. Playlist names, Xtream names, programme titles, and programme descriptions remain unchanged; persisted target data does not require rebuilding. | +| `force_redirect` | Bool | No | `false` | Optional redirect-related behavior switch. This influences how Tuliprox serves final stream delivery where redirect-style output handling is required by the deployment model. | > **Shared HLS:** `share_live_streams.hls` requires `reverse_proxy.hls_cache` in `config.yml`. > Start with [Shared HLS Sessions](./shared-hls-sessions.md) for the feature overview and @@ -1315,6 +1323,51 @@ targets: > If the reverse-proxy buffer size is increased above `1024`, memory usage increases accordingly. > Example: with a buffer size of `2048`, each shared channel consumes at least **24 MB**. +#### EPG Output Normalization + +`epg_output` applies to the entire target so every output format uses the same EPG identity space. Both options +default to `false`; when they are omitted, Tuliprox preserves the source casing and existing output behavior. + +```yaml +targets: + - name: sample_target + options: + epg_output: + lowercase_ids: true + lowercase_xmltv_display_names: true + output: + - type: m3u + - type: xtream +``` + +With both options enabled, neutral input values such as `Example.Channel` and `Sample Network` produce a canonical +technical ID of `example.channel` and the following XMLTV output: + +```xml + + sample network + + + Example Programme + +``` + +`lowercase_ids` uses ASCII lowercase for technical identifiers. Non-ASCII characters in those IDs remain unchanged. +The canonical visible ID is used consistently for M3U `tvg-id`, Xtream `epg_channel_id`, XMLTV `` and +``, and Short EPG / Stream EPG responses. Target EPG database keys and API lookup keys use the same +target output casing: source case is preserved while the option is disabled, and ASCII lowercase is used after the +option is enabled and the target is refreshed. This keeps existing mixed-case target databases compatible while the +two XMLTV attributes continue to use exactly the same visible value. + +`lowercase_xmltv_display_names` uses Unicode lowercase only while serializing XMLTV `` values. It takes +effect for subsequent XMLTV responses and normally does not require rebuilding persisted target data. It does not +change playlist or Xtream names, programme titles or descriptions, input data, parser values, caches, or Smart Match +behavior. + +Changing `lowercase_ids` requires a full target refresh so persisted playlist and EPG artifacts use the same visible +IDs; a configuration hot reload alone is insufficient. Clients may need to re-index their EPG data once after the +visible IDs change. + --- ### 3.2.7 Output Formats (`output`) diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 2b42c69c9..324d73d74 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -357,6 +357,10 @@ "CONFIG_TARGET_OPTIONS": { "SHARE_LIVE_STREAMS": "Enables live stream sharing. The master toggle sets HLS and MPEG-TS together; each format can also be changed separately." }, + "EPG_OUTPUT_OPTIONS": { + "LOWERCASE_IDS": "Canonicalizes technical EPG IDs to ASCII lowercase across M3U, Xtream, XMLTV, and EPG API output. A full target refresh is required after changing this option.", + "LOWERCASE_XMLTV_DISPLAY_NAMES": "Converts only XMLTV display-name text to Unicode lowercase. Playlist names and programme titles remain unchanged." + }, "CONFIG_TARGET_SHARE_LIVE_STREAMS": { "HLS": "Enables HLS live sharing for the new HLS cache proxy path.", "MPEG_TS": "Enables MPEG-TS live sharing for the existing stream-sharing path." @@ -1015,6 +1019,7 @@ "EPG": "Epg", "EDIT_EPG_SMART_MATCH": "Edit EPG Smart Match", "EPG_LOGO_OVERRIDE": "Logo Override", + "EPG_OUTPUT": "EPG output", "EPG_REQUEST_TIMESHIFT": "Epg Request Timeshift", "EPG_SMART_MATCH": "EPG Smart Match", "EPG_SOURCES": "EPG Sources", @@ -1102,6 +1107,8 @@ "LOG_ACTIVE_USER": "Log Active User", "LOG_CONFIG": "Log", "LOG_LEVEL": "Log Level", + "LOWERCASE_EPG_IDS": "Lowercase EPG IDs", + "LOWERCASE_XMLTV_DISPLAY_NAMES": "Lowercase XMLTV display names", "M3U": "m3u", "M3U_BATCH": "m3u batch", "MAIN": "Main", diff --git a/frontend/public/assets/i18n/ru.json b/frontend/public/assets/i18n/ru.json index fbc86ead3..602368fce 100644 --- a/frontend/public/assets/i18n/ru.json +++ b/frontend/public/assets/i18n/ru.json @@ -331,6 +331,10 @@ "USE_MEMORY_CACHE": "Если включено, плейлист кэшируется в оперативной памяти для уменьшения нагрузки на диск (увеличивает использование памяти).", "WATCH": "Список шаблонов групп для мониторинга изменений. Уведомления отправляются через настроенный мессенджер." }, + "EPG_OUTPUT_OPTIONS": { + "LOWERCASE_IDS": "Преобразует технические идентификаторы EPG в нижний регистр ASCII во всех выходах M3U, Xtream, XMLTV и API EPG. После изменения этой опции требуется полное обновление целевого плейлиста.", + "LOWERCASE_XMLTV_DISPLAY_NAMES": "Преобразует в нижний регистр Unicode только текст display-name XMLTV. Имена плейлиста и названия программ остаются без изменений." + }, "CONTENT_SECURITY_POLICY_CONFIG": { "CUSTOM_ATTRIBUTES": "Дополнительные атрибуты CSP, добавляемые к стандартной политике.", "ENABLED": "Включает или отключает заголовки Content Security Policy для веб-интерфейса." @@ -976,6 +980,7 @@ "EPG": "Epg", "EDIT_EPG_SMART_MATCH": "Редактировать интеллектуальное сопоставление EPG", "EPG_LOGO_OVERRIDE": "Замена логотипа", + "EPG_OUTPUT": "Вывод EPG", "EPG_REQUEST_TIMESHIFT": "Сдвиг времени запроса Epg", "EPG_SMART_MATCH": "Интеллектуальное сопоставление EPG", "EPG_SOURCES": "Источники EPG", @@ -1058,6 +1063,8 @@ "LOG_ACTIVE_USER": "Логировать активного пользователя", "LOG_CONFIG": "Лог", "LOG_LEVEL": "Уровень логирования", + "LOWERCASE_EPG_IDS": "Идентификаторы EPG в нижнем регистре", + "LOWERCASE_XMLTV_DISPLAY_NAMES": "Отображаемые имена XMLTV в нижнем регистре", "M3U": "m3u", "M3U_BATCH": "m3u batch", "MAIN": "Главное", diff --git a/frontend/src/app/components/source_editor/target_form.rs b/frontend/src/app/components/source_editor/target_form.rs index d2d8945eb..7e2852ed3 100644 --- a/frontend/src/app/components/source_editor/target_form.rs +++ b/frontend/src/app/components/source_editor/target_form.rs @@ -34,6 +34,9 @@ const LABEL_HLS: &str = "LABEL.HLS"; const LABEL_MPEG_TS: &str = "LABEL.MPEG_TS"; const LABEL_REMOVE_DUPLICATES: &str = "LABEL.REMOVE_DUPLICATES"; const LABEL_FORCE_REDIRECT: &str = "LABEL.FORCE_REDIRECT"; +const LABEL_EPG_OUTPUT: &str = "LABEL.EPG_OUTPUT"; +const LABEL_LOWERCASE_EPG_IDS: &str = "LABEL.LOWERCASE_EPG_IDS"; +const LABEL_LOWERCASE_XMLTV_DISPLAY_NAMES: &str = "LABEL.LOWERCASE_XMLTV_DISPLAY_NAMES"; const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; const LABEL_OPTIONS: &str = "LABEL.OPTIONS"; @@ -105,6 +108,8 @@ pub enum ConfigTargetOptionsFormAction { ShareLiveStreamsHls(bool), ShareLiveStreamsMpegTs(bool), RemoveDuplicates(bool), + LowercaseEpgIds(bool), + LowercaseXmltvDisplayNames(bool), ForceRedirect(Option), SetAll(ConfigTargetOptions), } @@ -137,6 +142,14 @@ impl yew::prelude::Reducible for ConfigTargetOptionsFormState { form.remove_duplicates = value; modified = true; } + ConfigTargetOptionsFormAction::LowercaseEpgIds(value) => { + form.epg_output.lowercase_ids = value; + modified = true; + } + ConfigTargetOptionsFormAction::LowercaseXmltvDisplayNames(value) => { + form.epg_output.lowercase_xmltv_display_names = value; + modified = true; + } ConfigTargetOptionsFormAction::ForceRedirect(value) => { form.force_redirect = value; modified = true; @@ -243,6 +256,32 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { } }; + let render_epg_output = + |readonly: bool, lowercase_ids_on_change: Callback, lowercase_names_on_change: Callback| { + html! { +
+
+ { translate.t(LABEL_EPG_OUTPUT) } +
+
+ { target_option_toggle( + translate.t(LABEL_LOWERCASE_EPG_IDS), + "EPG_OUTPUT_OPTIONS.LOWERCASE_IDS", + target_options_state.form.epg_output.lowercase_ids, + readonly, + lowercase_ids_on_change, + ) } + { target_option_toggle( + translate.t(LABEL_LOWERCASE_XMLTV_DISPLAY_NAMES), + "EPG_OUTPUT_OPTIONS.LOWERCASE_XMLTV_DISPLAY_NAMES", + target_options_state.form.epg_output.lowercase_xmltv_display_names, + readonly, + lowercase_names_on_change, + ) } +
+
+ } + }; if !props.allow_write { html! { @@ -274,6 +313,7 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { { config_field_bool!(target_options_state.form, translate.t(LABEL_REMOVE_DUPLICATES), remove_duplicates) } + { render_epg_output(true, Callback::noop(), Callback::noop()) }
{ translate.t(LABEL_FORCE_REDIRECT) } @@ -308,6 +348,18 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { target_options_state.dispatch(ConfigTargetOptionsFormAction::ShareLiveStreamsMpegTs(value)); }) }; + let lowercase_epg_ids_on_change = { + let target_options_state = target_options_state.clone(); + Callback::from(move |value| { + target_options_state.dispatch(ConfigTargetOptionsFormAction::LowercaseEpgIds(value)); + }) + }; + let lowercase_xmltv_display_names_on_change = { + let target_options_state = target_options_state.clone(); + Callback::from(move |value| { + target_options_state.dispatch(ConfigTargetOptionsFormAction::LowercaseXmltvDisplayNames(value)); + }) + }; html! {
@@ -338,6 +390,11 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html {
{ edit_field_bool!(target_options_state, translate.t(LABEL_REMOVE_DUPLICATES), remove_duplicates, ConfigTargetOptionsFormAction::RemoveDuplicates) } + { render_epg_output( + false, + lowercase_epg_ids_on_change, + lowercase_xmltv_display_names_on_change, + ) }
{ translate.t(LABEL_FORCE_REDIRECT) } @@ -502,3 +559,33 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html {
} } + +#[cfg(test)] +mod tests { + use super::{ConfigTargetOptionsFormAction, ConfigTargetOptionsFormState}; + use shared::model::ConfigTargetOptions; + use std::rc::Rc; + use yew::prelude::Reducible; + + fn default_options_state() -> Rc { + ConfigTargetOptionsFormState { form: ConfigTargetOptions::default(), modified: false }.into() + } + + #[test] + fn lowercase_epg_ids_action_updates_nested_option() { + let state = default_options_state().reduce(ConfigTargetOptionsFormAction::LowercaseEpgIds(true)); + + assert!(state.form.epg_output.lowercase_ids); + assert!(state.modified); + assert!(!state.form.epg_output.lowercase_xmltv_display_names); + } + + #[test] + fn lowercase_xmltv_display_names_action_updates_nested_option() { + let state = default_options_state().reduce(ConfigTargetOptionsFormAction::LowercaseXmltvDisplayNames(true)); + + assert!(state.form.epg_output.lowercase_xmltv_display_names); + assert!(state.modified); + assert!(!state.form.epg_output.lowercase_ids); + } +} diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index 98ea3201e..16d165a88 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -45,6 +45,20 @@ impl ConfigTargetShareLiveStreams { pub fn is_empty(&self) -> bool { !self.hls && !self.mpeg_ts } } +/// Controls optional canonicalization of EPG data emitted for a target. +#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct EpgOutputOptions { + #[serde(default, skip_serializing_if = "is_false")] + pub lowercase_ids: bool, + #[serde(default, skip_serializing_if = "is_false")] + pub lowercase_xmltv_display_names: bool, +} + +impl EpgOutputOptions { + pub const fn is_empty(&self) -> bool { !self.lowercase_ids && !self.lowercase_xmltv_display_names } +} + #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct ConfigTargetOptions { @@ -58,6 +72,8 @@ pub struct ConfigTargetOptions { pub share_live_streams: ConfigTargetShareLiveStreams, #[serde(default, skip_serializing_if = "is_false")] pub remove_duplicates: bool, + #[serde(default, skip_serializing_if = "EpgOutputOptions::is_empty")] + pub epg_output: EpgOutputOptions, #[serde(default, skip_serializing_if = "Option::is_none")] pub force_redirect: Option, } @@ -67,10 +83,15 @@ impl ConfigTargetOptions { !self.ignore_logo && self.share_live_streams.is_empty() && !self.remove_duplicates + && self.epg_output.is_empty() && (self.force_redirect.is_none() || self.force_redirect.is_some_and(|f| f.has_full_flags() || f.is_empty())) } + pub const fn lowercase_epg_ids(&self) -> bool { self.epg_output.lowercase_ids } + + pub const fn lowercase_xmltv_display_names(&self) -> bool { self.epg_output.lowercase_xmltv_display_names } + pub fn share_live_hls_enabled(&self) -> bool { self.share_live_streams.hls } pub fn share_live_mpeg_ts_enabled(&self) -> bool { self.share_live_streams.mpeg_ts } @@ -475,8 +496,8 @@ impl ConfigTargetDto { #[cfg(test)] mod tests { use super::{ - ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, M3uTargetOutputDto, StrmTargetOutputDto, - TargetOutputDto, XtreamTargetOutputDto, + ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, EpgOutputOptions, M3uTargetOutputDto, + StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto, }; fn target_with_outputs(output: Vec) -> ConfigTargetDto { @@ -589,6 +610,68 @@ share_live_streams: true ); } + #[test] + fn target_options_default_epg_output_is_disabled_and_omitted() { + let options = serde_saphyr::from_str::("{}") + .expect("target options without epg_output should deserialize"); + + assert!(!options.lowercase_epg_ids()); + assert!(!options.lowercase_xmltv_display_names()); + assert!(options.epg_output.is_empty()); + assert!(options.is_empty()); + + let serialized = serde_saphyr::to_string(&options).expect("default target options should serialize"); + assert!(!serialized.contains("epg_output"), "default epg_output should be omitted, got: {serialized}"); + } + + #[test] + fn target_options_epg_output_roundtrips() { + let yaml = r#" +epg_output: + lowercase_ids: true + lowercase_xmltv_display_names: true +"#; + + let options = + serde_saphyr::from_str::(yaml).expect("configured epg_output should deserialize"); + + assert!(options.lowercase_epg_ids()); + assert!(options.lowercase_xmltv_display_names()); + assert!(!options.is_empty()); + + let serialized = serde_saphyr::to_string(&options).expect("configured epg_output should serialize"); + let roundtripped = serde_saphyr::from_str::(&serialized) + .expect("serialized epg_output should deserialize"); + assert_eq!(roundtripped, options); + } + + #[test] + fn target_options_epg_output_makes_options_nonempty() { + let lowercase_ids = ConfigTargetOptions { + epg_output: EpgOutputOptions { lowercase_ids: true, ..EpgOutputOptions::default() }, + ..ConfigTargetOptions::default() + }; + let lowercase_display_names = ConfigTargetOptions { + epg_output: EpgOutputOptions { lowercase_xmltv_display_names: true, ..EpgOutputOptions::default() }, + ..ConfigTargetOptions::default() + }; + + assert!(!lowercase_ids.is_empty()); + assert!(!lowercase_display_names.is_empty()); + } + + #[test] + fn target_options_reject_unknown_epg_output_fields() { + let yaml = r#" +epg_output: + lowercase_id: true +"#; + + let result = serde_saphyr::from_str::(yaml); + + assert!(result.is_err(), "unknown epg_output fields must be rejected"); + } + #[test] fn target_options_mpeg_ts_helper_keeps_existing_stream_share_semantics() { let hls_only = ConfigTargetOptions {