diff --git a/CHANGELOG.md b/CHANGELOG.md index 50291744c..1d3cf7bf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,10 +106,15 @@ ## 🌟 New Features -- **Targets can require usable EPG data for live channels**: setting `options.required_epg: true` keeps only live - entries whose EPG ID resolves to programme data from an available EPG source. The normal target filter and mappings - still run first, so EPG matching operates on the already reduced playlist. VOD, series, catch-up, and local-library - entries are unaffected, and inputs without a successfully materialized EPG source keep their live entries. +- **Target filters can run during processing or immediately before persistence.** The existing scalar `filter` syntax + remains the `processing` stage. The staged map accepts optional `processing` and `persist` filters; `persist` sees + the fully finalized state after EPG processing, mappings, merge, deduplication, sorting, numbering, and counters. + Omitting `processing` no longer requires a match-all filter, and targets may omit `filter` entirely. Presence checks + use `IS EMPTY` / `IS NOT EMPTY`; `= EMPTY` / `!= EMPTY` remain accepted as compact aliases. + +- **Targets can clear invalid EPG IDs without removing playlist entries.** Setting + `options.clear_invalid_epg_ids: true` clears IDs that do not resolve to processed EPG data, including IDs changed by + mappings. Without the option, unmatched IDs are preserved. The old `required_epg` name remains a read-only alias. - **Ten events for the failures that used to be silent**: the registry described states nothing emitted, and several subsystems reported their start and their success but never their own failure. The taxonomy is now 42 events (up @@ -1197,9 +1202,9 @@ ## ⚙️ New Settings - **source.yml (target `options`)**: - - Added optional `required_epg` (`bool`, default `false`) to remove unmatched live entries after EPG matching. - The setting is evaluated independently for each target and does not affect VOD, series, catch-up, or local-library - entries. If an input has no successfully materialized EPG source, its live entries are left unchanged. + - Added optional `clear_invalid_epg_ids` (`bool`, default `false`) to clear unresolved live-channel EPG IDs after EPG + matching and final mappings without removing playlist entries. The legacy name `required_epg` is accepted while + reading existing configuration and is rewritten as `clear_invalid_epg_ids` when serialized. - **config.yml (`video.download.recording`)**: - Added `enabled` (`bool`, default `true`): master switch for the DVR. When `false` the REST routes answer diff --git a/Cargo.lock b/Cargo.lock index 232b70c22..bfc195aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4454,6 +4454,7 @@ name = "tuliprox-config-loader" version = "3.3.96" dependencies = [ "arc-swap", + "blake3", "chrono", "log", "serde", diff --git a/backend/app/src/api/api_utils.rs b/backend/app/src/api/api_utils.rs index 235681a42..bed3dd0bd 100644 --- a/backend/app/src/api/api_utils.rs +++ b/backend/app/src/api/api_utils.rs @@ -6232,7 +6232,7 @@ mod tests { ..ConfigTargetOptions::default() }), sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -7662,7 +7662,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -7721,7 +7721,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -7913,7 +7913,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -8064,7 +8064,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -8112,7 +8112,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -8186,7 +8186,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, @@ -8263,7 +8263,7 @@ mod tests { name: "test".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: Vec::new(), rename: None, mapping_ids: None, diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index 088c5c8b0..ab8c81483 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -610,7 +610,7 @@ mod tests { name: "target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::default(), trakt: None, diff --git a/backend/app/src/api/endpoints/download_api.rs b/backend/app/src/api/endpoints/download_api.rs index a77853863..4f5c1bfe7 100644 --- a/backend/app/src/api/endpoints/download_api.rs +++ b/backend/app/src/api/endpoints/download_api.rs @@ -907,29 +907,37 @@ fn spawn_recording_notification_after_persist( }); } +fn apply_requeue_active( + candidate: &mut crate::api::model::PersistedDownloadQueue, + uuid: &str, + state: DownloadState, + error: Option, + promote: bool, +) -> Option { + let mut download = candidate.active.take()?; + if download.uuid != uuid { + candidate.active = Some(download); + return None; + } + download.finished = false; + download.paused = false; + download.error = error; + download.state = state; + download.next_retry_at = None; + candidate.queue.insert(0, download); + if promote { + candidate.active = Some(candidate.queue.remove(0)); + } + Some(true) +} + async fn requeue_active_download_for_retry( download_queue: &DownloadQueue, uuid: &str, promote: bool, ) -> Result { Ok(mutate_optional(download_queue, |candidate| { - let Some(mut download) = candidate.active.take() else { - return Ok(None); - }; - if download.uuid != uuid { - candidate.active = Some(download); - return Ok(None); - } - download.finished = false; - download.paused = false; - download.error = None; - download.state = DownloadState::Queued; - download.next_retry_at = None; - candidate.queue.insert(0, download); - if promote { - candidate.active = Some(candidate.queue.remove(0)); - } - Ok(Some(true)) + Ok(apply_requeue_active(candidate, uuid, DownloadState::Queued, None, promote)) }) .await? .unwrap_or(false)) @@ -943,23 +951,7 @@ async fn requeue_active_download_for_capacity_wait( consumed_control: Option, ) -> Result { let mutation = |candidate: &mut crate::api::model::PersistedDownloadQueue| { - let Some(mut download) = candidate.active.take() else { - return Ok(None); - }; - if download.uuid != uuid { - candidate.active = Some(download); - return Ok(None); - } - download.finished = false; - download.paused = false; - download.error = Some(reason.to_string()); - download.state = DownloadState::WaitingForCapacity; - download.next_retry_at = None; - candidate.queue.insert(0, download); - if promote { - candidate.active = Some(candidate.queue.remove(0)); - } - Ok(Some(true)) + Ok(apply_requeue_active(candidate, uuid, DownloadState::WaitingForCapacity, Some(reason.to_string()), promote)) }; let result = if let Some(control) = consumed_control { download_queue.mutate_optional_and_clear_control(control, mutation).await? diff --git a/backend/app/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs index 86ba544f6..5dfcf55ab 100644 --- a/backend/app/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -288,7 +288,7 @@ async fn save_config_sources( } } - match crate::config_loader::persist_source_config(&app_state.app_config, None, sources).await { + match crate::config_loader::replace_source_config_from_user_edit(&app_state.app_config, None, sources).await { Ok(_) => {} Err(err) => { error!("Failed to persist source.yml {err}"); diff --git a/backend/app/src/api/endpoints/v1_api_playlist.rs b/backend/app/src/api/endpoints/v1_api_playlist.rs index d7b5eca9b..51b3caec0 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist.rs @@ -1245,7 +1245,7 @@ mod tests { name: "stable-target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![], rename: None, mapping_ids: None, @@ -1354,7 +1354,7 @@ mod tests { name: name.to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![], rename: None, mapping_ids: None, @@ -1566,7 +1566,7 @@ mod tests { name: "target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![], rename: None, mapping_ids: None, @@ -1631,7 +1631,7 @@ mod tests { name: "target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![], rename: None, mapping_ids: None, @@ -1682,7 +1682,7 @@ mod tests { name: "target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![], rename: None, mapping_ids: None, diff --git a/backend/app/src/api/endpoints/xmltv_api.rs b/backend/app/src/api/endpoints/xmltv_api.rs index 5c8a0339e..0bf75440b 100644 --- a/backend/app/src/api/endpoints/xmltv_api.rs +++ b/backend/app/src/api/endpoints/xmltv_api.rs @@ -979,7 +979,7 @@ mod tests { name: "mixed-target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![ TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), @@ -1011,7 +1011,7 @@ mod tests { name: "xtream-only".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, diff --git a/backend/app/src/api/endpoints/xtream_api.rs b/backend/app/src/api/endpoints/xtream_api.rs index 81036a84f..9fd0089f1 100644 --- a/backend/app/src/api/endpoints/xtream_api.rs +++ b/backend/app/src/api/endpoints/xtream_api.rs @@ -2086,7 +2086,7 @@ mod tests { name: "ics-xtream".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: Filter::default().into(), output: vec![TargetOutput::Xtream(XtreamTargetOutput { flags: XtreamTargetFlagsSet::new(), trakt: None, diff --git a/backend/app/src/api/hdhomerun_proprietary.rs b/backend/app/src/api/hdhomerun_proprietary.rs index ff7cf9a17..b8b2c412d 100644 --- a/backend/app/src/api/hdhomerun_proprietary.rs +++ b/backend/app/src/api/hdhomerun_proprietary.rs @@ -41,32 +41,52 @@ mod packet { // --- UDP Discovery Logic --- -fn write_tlv_u8(buf: &mut bytes::BytesMut, tag: u8, value: u8) { - buf.put_u8(tag); - write_tlv_length(buf, 1); - buf.put_u8(value); +trait TlvValue { + fn write_tlv_payload(self, buf: &mut BytesMut); } -fn write_tlv_u32(buf: &mut bytes::BytesMut, tag: u8, value: u32) { - buf.put_u8(tag); - write_tlv_length(buf, 4); - buf.put_u32(value); -} - -fn write_tlv_str(buf: &mut bytes::BytesMut, tag: u8, value: &str) { - let bytes = value.as_bytes(); - if bytes.len() > 0x7FFF { - // maximum length for 15-bit TLV = 32767 - log::warn!("TLV string too long, truncating to 32767 bytes"); +impl TlvValue for u8 { + #[inline] + fn write_tlv_payload(self, buf: &mut BytesMut) { + write_tlv_length(buf, 1); + buf.put_u8(self); } - let len = std::cmp::min(bytes.len(), 0x7FFF); +} + +impl TlvValue for u32 { + #[inline] + fn write_tlv_payload(self, buf: &mut BytesMut) { + write_tlv_length(buf, 4); + buf.put_u32(self); + } +} + +impl TlvValue for &str { + #[inline] + fn write_tlv_payload(self, buf: &mut BytesMut) { + let bytes = self.as_bytes(); + if bytes.len() > 0x7FFF { + // maximum length for 15-bit TLV = 32767 + log::warn!("TLV string too long, truncating to 32767 bytes"); + } + let len = std::cmp::min(bytes.len(), 0x7FFF); + write_tlv_length(buf, len); + buf.put_slice(&bytes[..len]); + } +} + +impl TlvValue for &String { + #[inline] + fn write_tlv_payload(self, buf: &mut BytesMut) { self.as_str().write_tlv_payload(buf); } +} + +fn write_tlv(buf: &mut BytesMut, tag: u8, value: T) { buf.put_u8(tag); - write_tlv_length(buf, len); - buf.put_slice(&bytes[..len]); + value.write_tlv_payload(buf); } // helper function for variable-length TLV -fn write_tlv_length(buf: &mut bytes::BytesMut, len: usize) { +fn write_tlv_length(buf: &mut BytesMut, len: usize) { if len <= 0x7F { // ≤ 127 -> 1 Byte length buf.put_u8(u8::try_from(len).unwrap_or(0xFF)); @@ -87,11 +107,11 @@ fn build_discover_response(device: &HdHomeRunDeviceConfig, server_host: &str) -> let device_id = u32::from_str_radix(&device.device_id, 16).unwrap_or(0); - write_tlv_u32(&mut payload, packet::HDHOMERUN_TAG_DEVICE_TYPE, packet::HDHOMERUN_DEVICE_TYPE_TUNER); - write_tlv_u32(&mut payload, packet::HDHOMERUN_TAG_DEVICE_ID, device_id); - write_tlv_str(&mut payload, packet::HDHOMERUN_TAG_BASE_URL, &base_url); - write_tlv_u8(&mut payload, packet::HDHOMERUN_TAG_TUNER_COUNT, device.tuner_count); - write_tlv_str(&mut payload, packet::HDHOMERUN_TAG_LINEUP_URL, &lineup_url); + write_tlv(&mut payload, packet::HDHOMERUN_TAG_DEVICE_TYPE, packet::HDHOMERUN_DEVICE_TYPE_TUNER); + write_tlv(&mut payload, packet::HDHOMERUN_TAG_DEVICE_ID, device_id); + write_tlv(&mut payload, packet::HDHOMERUN_TAG_BASE_URL, &base_url); + write_tlv(&mut payload, packet::HDHOMERUN_TAG_TUNER_COUNT, device.tuner_count); + write_tlv(&mut payload, packet::HDHOMERUN_TAG_LINEUP_URL, &lineup_url); let mut response = BytesMut::new(); response.put_u16(packet::HDHOMERUN_TYPE_DISCOVER_RSP); @@ -315,11 +335,11 @@ async fn process_getset_request(request: &[u8], app_state: &Arc) -> Ve trace!("Received GET/SET for: {name}"); let name_str = name.trim_end_matches('\0'); - write_tlv_str(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_NAME, name_str); + write_tlv(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_NAME, name_str); match name_str { "/sys/model" => { - write_tlv_str(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, "hdhomerun4_atsc"); + write_tlv(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, "hdhomerun4_atsc"); } s if s.starts_with("/tuner") && s.ends_with("/status") => { let rest = &s[6..]; @@ -331,7 +351,7 @@ async fn process_getset_request(request: &[u8], app_state: &Arc) -> Ve } else { "ch=none lock=none ss=0 snq=0 seq=0 bps=0 pps=0".to_string() }; - write_tlv_str(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, &status_str); + write_tlv(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, &status_str); } } s if s.starts_with("/tuner") && s.ends_with("/vchannel") => { @@ -344,15 +364,15 @@ async fn process_getset_request(request: &[u8], app_state: &Arc) -> Ve } else { "none".intern() }; - write_tlv_str(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, &vchannel); + write_tlv(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, vchannel.as_ref()); } } s if s.starts_with("/tuner") && s.ends_with("/lockkey") => { let err_msg = "ERROR: resource locked"; - write_tlv_str(&mut response_payload, packet::HDHOMERUN_TAG_ERROR_MESSAGE, err_msg); + write_tlv(&mut response_payload, packet::HDHOMERUN_TAG_ERROR_MESSAGE, err_msg); } _ => { - write_tlv_str(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, ""); + write_tlv(&mut response_payload, packet::HDHOMERUN_TAG_GETSET_VALUE, ""); } } } diff --git a/backend/app/src/api/mod.rs b/backend/app/src/api/mod.rs index 94d427d15..f2438acf5 100644 --- a/backend/app/src/api/mod.rs +++ b/backend/app/src/api/mod.rs @@ -4,6 +4,7 @@ mod http_layers; pub mod model; mod panel_api; pub mod setup_api; +mod source_yml_patch; pub(crate) use panel_api::sync_panel_api_exp_dates; mod config_file; mod endpoints; diff --git a/backend/app/src/api/panel_api.rs b/backend/app/src/api/panel_api.rs index f91b91f64..51e818995 100644 --- a/backend/app/src/api/panel_api.rs +++ b/backend/app/src/api/panel_api.rs @@ -26,16 +26,18 @@ use crate::{ create_panel_api_provisioning_stream_with_stop, create_provider_connections_exhausted_stream, AppState, StreamDetails, }, + source_yml_patch::{ + derive_unique_alias_name, derive_unique_alias_name_set, execute_source_yml_patches, + resolve_provisioned_account_base_url, SourcesYmlPatch, + }, }, - config_loader::{persist_source_config_preserving_templates, read_sources_file_from_path}, model::{ - is_input_expired, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, PanelApiConfig, - PanelApiQueryParam, ProxyUserCredentials, + is_input_expired, is_input_expired_at, ConfigInput, ConfigInputAlias, GracePeriodOptions, InputSource, + PanelApiConfig, PanelApiQueryParam, ProxyUserCredentials, }, repository::{ - compare_alias_exp_date_with_order, csv_patch_batch_append, csv_patch_batch_remove_expired, - csv_patch_batch_sort_by_exp_date, csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, - get_csv_file_path, AliasExpDateSortOrder, + csv_patch_batch_append, csv_patch_batch_remove_expired, csv_patch_batch_sort_by_exp_date, + csv_patch_batch_update_credentials, csv_patch_batch_update_exp_date, get_csv_file_path, AliasExpDateSortOrder, }, utils::{debug_if_enabled, format_http_status, request}, }; @@ -50,12 +52,11 @@ use shared::{ concat_string, create_bitset, error::{string_to_io_error, TuliproxError}, model::{ - ConfigInputAliasDto, ConfigInputDto, DisconnectReason, InputType, PanelApiAliasPoolSizeValue, - PanelApiProvisioningMethod, ProxyUserStatus, SourcesConfigDto, VirtualId, + DisconnectReason, InputType, PanelApiAliasPoolSizeValue, PanelApiProvisioningMethod, ProxyUserStatus, VirtualId, }, utils::{ get_base_url_from_str, get_credentials_from_url, get_credentials_from_url_str, get_i64_from_serde_value, - get_string_from_serde_value, parse_timestamp, sanitize_sensitive_info, Internable, PROVIDER_SCHEME_PREFIX, + get_string_from_serde_value, parse_timestamp, sanitize_sensitive_info, }, }; use smallvec::SmallVec; @@ -202,13 +203,6 @@ fn normalize_panel_expire(value: &str, ctx: Option<&PanelApiTimeContext>) -> Opt } } -fn is_input_expired_at(exp_date: Option, now: u64) -> bool { - let Some(exp_date) = exp_date else { - return false; - }; - u64::try_from(exp_date).map_or(true, |exp_ts| exp_ts <= now) -} - fn is_expiring_with_offset_at(exp_date: Option, offset_secs: u64, now: u64) -> bool { let Some(exp_date) = exp_date else { return false; @@ -1126,15 +1120,17 @@ fn collect_accounts(input: &ConfigInput) -> Vec { } fn compare_alias_exp_date_config(a: &ConfigInputAlias, b: &ConfigInputAlias) -> Ordering { - let a_ts = a.exp_date.unwrap_or(i64::MIN); - let b_ts = b.exp_date.unwrap_or(i64::MIN); - b_ts.cmp(&a_ts).then_with(|| a.name.cmp(&b.name)) + compare_named_exp_date(a.exp_date, a.name.as_ref(), b.exp_date, b.name.as_ref()) } fn compare_account_exp_date(a: &AccountCredentials, b: &AccountCredentials) -> Ordering { - let a_ts = a.exp_date.unwrap_or(i64::MIN); - let b_ts = b.exp_date.unwrap_or(i64::MIN); - b_ts.cmp(&a_ts).then_with(|| a.name.cmp(&b.name)) + compare_named_exp_date(a.exp_date, a.name.as_ref(), b.exp_date, b.name.as_ref()) +} + +fn compare_named_exp_date(a_exp_date: Option, a_name: &str, b_exp_date: Option, b_name: &str) -> Ordering { + let a_ts = a_exp_date.unwrap_or(i64::MIN); + let b_ts = b_exp_date.unwrap_or(i64::MIN); + b_ts.cmp(&a_ts).then_with(|| a_name.cmp(b_name)) } fn aliases_need_sort_config(aliases: &[ConfigInputAlias]) -> bool { @@ -1144,20 +1140,6 @@ fn aliases_need_sort_config(aliases: &[ConfigInputAlias]) -> bool { aliases.windows(2).any(|pair| compare_alias_exp_date_config(&pair[0], &pair[1]) == Ordering::Greater) } -fn sort_aliases_by_exp_date_order(aliases: &mut Vec, order: AliasExpDateSortOrder) -> bool { - if aliases.len() < 2 { - return false; - } - let mut sorted = aliases.clone(); - sorted.sort_by(|a, b| compare_alias_exp_date_with_order(a, b, order)); - if &sorted == aliases { - false - } else { - *aliases = sorted; - true - } -} - fn sort_account_aliases_keep_root_first(accounts: &mut Vec, root_name: &str) { let root = accounts.iter().find(|acct| acct.name.as_ref() == root_name).cloned(); let mut aliases: Vec = @@ -1202,39 +1184,17 @@ fn should_reload_sources_after_internal_write(app_state: &AppState) -> bool { !app_state.app_config.config.load().config_hot_reload } -fn append_sources_yml_alias( - input_name: &Arc, - input: &mut ConfigInputDto, - alias_name: Arc, - base_url: String, - username: String, - password: String, - exp_date: Option, -) -> Result { - let input_type = input.input_type; - let aliases = input.aliases.get_or_insert_with(Vec::new); - let next_index = aliases.iter().map(|alias| alias.id).max().unwrap_or(0); - if next_index == u16::MAX { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: cannot add alias for '{input_name}': alias id overflow" - ))); - } - - let mut alias = ConfigInputAliasDto { - id: 0, - name: alias_name, - url: base_url, - username: Some(username), - password: Some(password), - priority: 0, - max_connections: 1, - exp_date, - enabled: true, - stalker: None, +fn resolve_batch_alias_path(batch_url: Option<&str>) -> Result, TuliproxError> { + let Some(batch_url) = batch_url.filter(|url| !url.trim().is_empty()) else { + return Ok(None); }; - alias.prepare(next_index, &input_type)?; - aliases.push(alias); - Ok(aliases.len().saturating_sub(1)) + get_csv_file_path(batch_url).map(Some).map_err(|err| TuliproxError::ConfigInput(format!("{err}"))) +} + +fn require_batch_alias_path(input: &ConfigInput) -> Result { + resolve_batch_alias_path(input.t_batch_url.as_deref())?.ok_or_else(|| { + TuliproxError::ConfigInput(format!("batch input '{}' does not define a CSV alias path", input.name)) + }) } pub(crate) fn is_alias_pool_max_reached(app_state: &AppState, input: &ConfigInput) -> bool { @@ -1301,414 +1261,6 @@ pub(crate) fn find_input_by_provider_name(app_state: &AppState, provider_name: & None } -#[derive(Debug, Clone)] -enum SourcesYmlPatch { - UpdatePanelApiCredits { - input_name: Arc, - credits: String, - }, - SortAliases { - input_name: Arc, - order: AliasExpDateSortOrder, - }, - UpdateExpDate { - input_name: Arc, - account_name: Arc, - exp_date: i64, - }, - UpdateRootCredentials { - input_name: Arc, - username: String, - password: String, - exp_date: Option, - }, - PersistProvisionedAccount { - input_name: Arc, - username: String, - password: String, - exp_date: Option, - }, - UpdateAliasCredentials { - input_name: Arc, - alias_name: Arc, - username: String, - password: String, - exp_date: Option, - }, - AddAlias { - input_name: Arc, - alias_name: Arc, - base_url: String, - username: String, - password: String, - exp_date: Option, - }, - RemoveExpiredAliases { - input_name: Arc, - }, -} - -fn update_url_query_credentials_if_present(url: &mut String, username: &str, password: &str) { - let Ok(mut parsed) = Url::parse(url.as_str()) else { - return; - }; - let mut pairs: Vec<(String, String)> = parsed.query_pairs().map(|(k, v)| (k.to_string(), v.to_string())).collect(); - let mut has_user = false; - let mut has_pass = false; - for (k, v) in &mut pairs { - if k.eq_ignore_ascii_case("username") { - *v = username.to_string(); - has_user = true; - } else if k.eq_ignore_ascii_case("password") { - *v = password.to_string(); - has_pass = true; - } - } - if has_user || has_pass { - if !has_user { - pairs.push(("username".to_string(), username.to_string())); - } - if !has_pass { - pairs.push(("password".to_string(), password.to_string())); - } - parsed.query_pairs_mut().clear(); - { - let mut qp = parsed.query_pairs_mut(); - for (k, v) in pairs { - qp.append_pair(k.as_str(), v.as_str()); - } - } - *url = parsed.to_string(); - } -} - -fn resolve_provisioned_account_base_url( - input_url: &str, - base_url_from_response: Option<&str>, - username: &str, - password: &str, -) -> String { - if input_url.starts_with(PROVIDER_SCHEME_PREFIX) { - let mut provider_url = input_url.to_string(); - update_url_query_credentials_if_present(&mut provider_url, username, password); - return provider_url; - } - - let base_url = base_url_from_response.map_or_else(|| input_url.to_string(), ToString::to_string); - if let Some(origin) = get_base_url_from_str(base_url.as_str()) { - let trimmed_origin = origin.trim(); - if !trimmed_origin.is_empty() && !trimmed_origin.eq_ignore_ascii_case("null") { - return origin; - } - } - - let trimmed_base = base_url.trim(); - if !trimmed_base.is_empty() && !trimmed_base.eq_ignore_ascii_case("null") { - return base_url; - } - - input_url.to_string() -} - -#[allow(clippy::too_many_lines)] -fn apply_sources_yml_patches(doc: &mut SourcesConfigDto, patches: &[SourcesYmlPatch]) -> Result { - if patches.is_empty() { - return Ok(false); - } - - let mut changed = false; - let mut inputs_by_name: HashMap, usize> = HashMap::with_capacity(doc.inputs.len()); - let mut alias_indices: Vec, usize>> = Vec::with_capacity(doc.inputs.len()); - for (idx, input) in doc.inputs.iter().enumerate() { - inputs_by_name.insert(input.name.clone(), idx); - let map = input - .aliases - .as_ref() - .map(|aliases| { - aliases - .iter() - .enumerate() - .map(|(idx, alias)| (alias.name.clone(), idx)) - .collect::, usize>>() - }) - .unwrap_or_default(); - alias_indices.push(map); - } - - for patch in patches { - match patch { - SourcesYmlPatch::UpdatePanelApiCredits { input_name, credits } => { - let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let Some(panel_api) = doc.inputs[idx].panel_api.as_mut() else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find panel_api for input '{input_name}' in source.yml" - ))); - }; - if panel_api.credits.as_deref().map(str::trim) != Some(credits.trim()) { - panel_api.credits = Some(credits.trim().to_string()); - changed = true; - } - } - SourcesYmlPatch::SortAliases { input_name, order } => { - let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { - continue; - }; - if sort_aliases_by_exp_date_order(aliases, *order) { - alias_indices[idx] = - aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); - changed = true; - } - } - SourcesYmlPatch::UpdateExpDate { input_name, account_name, exp_date } => { - let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - if account_name == input_name { - if doc.inputs[idx].exp_date != Some(*exp_date) - || !doc.inputs[idx].enabled - || doc.inputs[idx].max_connections != 1 - { - doc.inputs[idx].exp_date = Some(*exp_date); - doc.inputs[idx].enabled = true; - doc.inputs[idx].max_connections = 1; - changed = true; - } - continue; - } - let Some(alias_idx) = alias_indices[idx].get(account_name).copied() else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find alias '{account_name}' under input '{input_name}' in source.yml" - ))); - }; - let aliases = doc.inputs[idx].aliases.as_mut().ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")) - })?; - if aliases[alias_idx].exp_date != Some(*exp_date) || aliases[alias_idx].max_connections != 1 { - aliases[alias_idx].exp_date = Some(*exp_date); - aliases[alias_idx].max_connections = 1; - changed = true; - } - } - SourcesYmlPatch::UpdateRootCredentials { input_name, username, password, exp_date } => { - let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let input = &mut doc.inputs[idx]; - let exp_date_changed = exp_date.is_some() && input.exp_date != *exp_date; - if input.username.as_deref() != Some(username.as_str()) - || input.password.as_deref() != Some(password.as_str()) - || exp_date_changed - { - input.username = Some(username.clone()); - input.password = Some(password.clone()); - input.enabled = true; - input.max_connections = 1; - if let Some(exp_date) = *exp_date { - input.exp_date = Some(exp_date); - } - update_url_query_credentials_if_present(&mut input.url, username, password); - changed = true; - } - } - SourcesYmlPatch::PersistProvisionedAccount { input_name, username, password, exp_date } => { - let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let input = &mut doc.inputs[idx]; - let current_root_is_usable = - input.exp_date.is_some() && !is_input_expired_at(input.exp_date, get_current_timestamp()); - if current_root_is_usable { - let mut existing_names = vec![input.name.clone()]; - if let Some(aliases) = input.aliases.as_ref() { - existing_names.extend(aliases.iter().map(|alias| alias.name.clone())); - } - let alias_name = derive_unique_alias_name(&existing_names, &input.name, username); - let base_url = resolve_provisioned_account_base_url(input.url.as_str(), None, username, password); - let alias_idx = append_sources_yml_alias( - input_name, - input, - Arc::clone(&alias_name), - base_url, - username.clone(), - password.clone(), - *exp_date, - )?; - alias_indices[idx].insert(Arc::clone(&alias_name), alias_idx); - if let Some(aliases) = input.aliases.as_mut() { - if sort_aliases_by_exp_date_order(aliases, AliasExpDateSortOrder::NewestFirst) { - alias_indices[idx] = - aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); - } - } - debug_if_enabled!( - "panel_api preserved current root for input {} and added provisioned account as alias {}", - sanitize_sensitive_info(input_name.as_ref()), - sanitize_sensitive_info(alias_name.as_ref()) - ); - changed = true; - continue; - } - debug_if_enabled!( - "panel_api stored provisioned account as root for input {} because current root is expired or missing exp_date", - sanitize_sensitive_info(input_name.as_ref()) - ); - let exp_date_changed = exp_date.is_some() && input.exp_date != *exp_date; - if input.username.as_deref() != Some(username.as_str()) - || input.password.as_deref() != Some(password.as_str()) - || exp_date_changed - { - input.username = Some(username.clone()); - input.password = Some(password.clone()); - input.enabled = true; - input.max_connections = 1; - if let Some(exp_date) = *exp_date { - input.exp_date = Some(exp_date); - } - update_url_query_credentials_if_present(&mut input.url, username, password); - changed = true; - } - } - SourcesYmlPatch::UpdateAliasCredentials { input_name, alias_name, username, password, exp_date } => { - let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let Some(alias_idx) = alias_indices[idx].get(alias_name).copied() else { - return Err(TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find alias '{alias_name}' under input '{input_name}' in source.yml" - ))); - }; - let aliases = doc.inputs[idx].aliases.as_mut().ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!("panel_api: input '{input_name}' has no aliases")) - })?; - let alias = &mut aliases[alias_idx]; - let exp_date_changed = exp_date.is_some() && alias.exp_date != *exp_date; - if alias.username.as_deref() != Some(username.as_str()) - || alias.password.as_deref() != Some(password.as_str()) - || exp_date_changed - { - alias.username = Some(username.clone()); - alias.password = Some(password.clone()); - alias.max_connections = 1; - if let Some(exp_date) = *exp_date { - alias.exp_date = Some(exp_date); - } - update_url_query_credentials_if_present(&mut alias.url, username, password); - changed = true; - } - } - SourcesYmlPatch::AddAlias { input_name, alias_name, base_url, username, password, exp_date } => { - let idx = *inputs_by_name.get(input_name).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let alias_idx = append_sources_yml_alias( - input_name, - &mut doc.inputs[idx], - Arc::clone(alias_name), - base_url.clone(), - username.clone(), - password.clone(), - *exp_date, - )?; - alias_indices[idx].insert(Arc::clone(alias_name), alias_idx); - changed = true; - } - SourcesYmlPatch::RemoveExpiredAliases { input_name } => { - let idx = *inputs_by_name.get(input_name).ok_or_else(|| { - TuliproxError::ConfigPanelApi(format!( - "panel_api: could not find input '{input_name}' in source.yml" - )) - })?; - let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { - continue; - }; - let before = aliases.len(); - aliases.retain(|a| !is_input_expired(a.exp_date)); - if aliases.len() != before { - alias_indices[idx] = - aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); - changed = true; - } - } - } - } - - Ok(changed) -} - -async fn persist_sources_yml_with_patches( - app_state: &Arc, - sources_path: &Path, - patches: &[SourcesYmlPatch], -) -> Result { - if patches.is_empty() { - return Ok(false); - } - let mut sources = read_sources_file_from_path(sources_path, false, false, None) - .await - .map_err(|e| TuliproxError::ConfigPanelApi(format!("panel_api: failed to read source file: {e}")))?; - - let changed = apply_sources_yml_patches(&mut sources, patches)?; - if !changed { - return Ok(false); - } - - persist_source_config_preserving_templates(&app_state.app_config, Some(sources_path), sources).await?; - Ok(true) -} - -const MAX_ALIAS_NAME_ATTEMPTS: usize = 1000; - -fn derive_unique_alias_name(existing: &[Arc], input_name: &Arc, username: &str) -> Arc { - let base: Arc = concat_string!(input_name, "-", username).intern(); - if !existing.contains(&base) { - return base; - } - for i in 2..MAX_ALIAS_NAME_ATTEMPTS { - let cand = concat_string!(&*base, "-", &i.to_string()).intern(); - if !existing.contains(&cand) { - return cand; - } - } - warn!("derive_unique_alias_name: exhausted {MAX_ALIAS_NAME_ATTEMPTS} attempts for base '{base}'; returning potentially duplicate name"); - base -} - -fn derive_unique_alias_name_set(existing: &HashSet>, input_name: &Arc, username: &str) -> String { - let base = format!("{input_name}-{username}"); - if !existing.contains(base.as_str()) { - return base; - } - for i in 2..MAX_ALIAS_NAME_ATTEMPTS { - let cand = format!("{base}-{i}"); - if !existing.contains(cand.as_str()) { - return cand; - } - } - warn!( - "derive_unique_alias_name_set: exhausted {MAX_ALIAS_NAME_ATTEMPTS} attempts for base '{base}'; returning potentially duplicate name" - ); - base -} - #[derive(Debug, Clone)] pub(crate) enum PanelApiProvisionOutcome { Renewed, @@ -1787,35 +1339,36 @@ async fn try_renew_expired_account( if let Some(new_exp) = refreshed_exp.or(acct.exp_date) { if is_batch { - let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); - if let Ok(csv_path) = get_csv_file_path(batch_url) { - let _csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; - if let Err(err) = csv_patch_batch_update_exp_date( - input.input_type, - &csv_path, - &acct.name, - &acct.username, - &acct.password, - new_exp, - ) - .await - { - debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); - } - if let Err(err) = csv_patch_batch_sort_by_exp_date( - input.input_type, - &csv_path, - AliasExpDateSortOrder::NewestFirst, - ) - .await - { - debug_if_enabled!("panel_api failed to sort csv accounts after renew: {}", err); + match require_batch_alias_path(input) { + Ok(csv_path) => { + let _csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; + if let Err(err) = csv_patch_batch_update_exp_date( + input.input_type, + &csv_path, + &acct.name, + &acct.username, + &acct.password, + new_exp, + ) + .await + { + debug_if_enabled!("panel_api failed to persist renew exp_date to csv: {}", err); + } + if let Err(err) = csv_patch_batch_sort_by_exp_date( + input.input_type, + &csv_path, + AliasExpDateSortOrder::NewestFirst, + ) + .await + { + debug_if_enabled!("panel_api failed to sort csv accounts after renew: {}", err); + } } + Err(err) => debug_if_enabled!("panel_api cannot resolve batch csv path: {}", err), } } else { - let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; let patches = [ - SourcesYmlPatch::UpdateExpDate { + SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: input.name.clone(), account_name: Arc::clone(&acct.name), exp_date: new_exp, @@ -1825,7 +1378,9 @@ async fn try_renew_expired_account( order: AliasExpDateSortOrder::NewestFirst, }, ]; - if let Err(err) = persist_sources_yml_with_patches(app_state, sources_path, &patches).await { + if let Err(err) = + execute_source_yml_patches(&app_state.app_config, sources_path, &patches).await + { debug_if_enabled!("panel_api failed to persist renew exp_date to source.yml: {}", err); } } @@ -1967,8 +1522,7 @@ async fn try_refresh_root_account_on_exhausted( let exp_date = if credentials_changed { refreshed_exp_date } else { refreshed_exp_date.or(input.exp_date) }; if is_batch { - let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); - let Ok(csv_path) = get_csv_file_path(batch_url) else { + let Ok(csv_path) = require_batch_alias_path(input) else { return None; }; let _csv_lock = app_state.app_config.file_locks.write_lock(&csv_path).await; @@ -2008,14 +1562,13 @@ async fn try_refresh_root_account_on_exhausted( return None; } } else { - let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; let patch = SourcesYmlPatch::PersistProvisionedAccount { input_name: input.name.clone(), username: active_username, password: active_password, exp_date, }; - if let Err(err) = persist_sources_yml_with_patches(app_state, sources_path, &[patch]).await { + if let Err(err) = execute_source_yml_patches(&app_state.app_config, sources_path, &[patch]).await { debug_if_enabled!("panel_api failed to persist root provisioning to source.yml: {}", err); return None; } @@ -2089,8 +1642,7 @@ async fn try_create_new_account( } if is_batch { - let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); - match get_csv_file_path(batch_url) { + match require_batch_alias_path(input) { Ok(csv_path) => { let batch_type = if input.input_type == InputType::Xtream { InputType::XtreamBatch @@ -2122,15 +1674,14 @@ async fn try_create_new_account( } Err(err) => { warn!( - "panel_api cannot resolve batch csv path {}: {}", - sanitize_sensitive_info(batch_url), - err + "panel_api cannot resolve batch csv path for {}: {}", + sanitize_sensitive_info(input.name.as_ref()), + err, ); return None; } } } else { - let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; let patches = [ SourcesYmlPatch::AddAlias { input_name: input.name.clone(), @@ -2145,7 +1696,7 @@ async fn try_create_new_account( order: AliasExpDateSortOrder::NewestFirst, }, ]; - if let Err(err) = persist_sources_yml_with_patches(app_state, sources_path, &patches).await { + if let Err(err) = execute_source_yml_patches(&app_state.app_config, sources_path, &patches).await { warn!("panel_api failed to persist new alias to source.yml: {err}"); return None; } @@ -2359,7 +1910,7 @@ async fn ensure_alias_pool_min( changed = true; } } else { - sources_yml_patches.push(SourcesYmlPatch::UpdateExpDate { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: input.name.clone(), account_name: acct.name.clone(), exp_date: new_exp, @@ -2493,9 +2044,17 @@ async fn sync_panel_api_for_input_on_boot( let _input_lock = app_state.app_config.file_locks.write_lock_str(format!("panel_api:{input_name}").as_str()).await; let mut any_change = false; - let is_batch = input.t_batch_url.as_ref().is_some_and(|u| !u.trim().is_empty()); - let batch_url = input.t_batch_url.as_deref().unwrap_or_default(); - let csv_path = if is_batch { get_csv_file_path(batch_url).ok() } else { None }; + let csv_path = match resolve_batch_alias_path(input.t_batch_url.as_deref()) { + Ok(path) => path, + Err(err) => { + warn!( + "panel_api boot sync skipped alias mutations for batch input {}: {}", + sanitize_sensitive_info(&input.name), + sanitize_sensitive_info(&err.to_string()) + ); + return false; + } + }; let mut sources_yml_patches: Vec = Vec::new(); let mut pending_sources_yml = false; let mut source_yml_sort_aliases_requested = false; @@ -2647,7 +2206,7 @@ async fn sync_panel_api_for_input_on_boot( } any_change = true; } else { - sources_yml_patches.push(SourcesYmlPatch::UpdateExpDate { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: input.name.clone(), account_name: acct.name.clone(), exp_date: new_exp, @@ -3065,7 +2624,7 @@ async fn sync_panel_api_for_input_on_boot( exp_date: Some(new_exp), }); } else { - sources_yml_patches.push(SourcesYmlPatch::UpdateExpDate { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: input.name.clone(), account_name: input.name.clone(), exp_date: new_exp, @@ -3498,7 +3057,7 @@ async fn sync_panel_api_for_input_on_boot( exp_date: Some(new_exp), }); } else { - sources_yml_patches.push(SourcesYmlPatch::UpdateExpDate { + sources_yml_patches.push(SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: input.name.clone(), account_name: account_name.clone(), exp_date: new_exp, @@ -3639,8 +3198,7 @@ async fn sync_panel_api_for_input_on_boot( } if pending_sources_yml { - let _src_lock = app_state.app_config.file_locks.write_lock(sources_path).await; - match persist_sources_yml_with_patches(app_state, sources_path, &sources_yml_patches).await { + match execute_source_yml_patches(&app_state.app_config, sources_path, &sources_yml_patches).await { Ok(true) => any_change = true, Ok(false) => {} Err(err) => debug_if_enabled!("panel_api boot sync failed to persist source.yml patches: {}", err), @@ -4166,12 +3724,15 @@ pub fn create_panel_api_provisioning_stream_details( #[cfg(test)] mod tests { use super::{ - apply_sources_yml_patches, build_panel_api_probe_targets, build_user_api_account_info_input_source, - panel_api_retry_after_from_header_value, panel_api_retryable_status, resolve_provisioned_account_base_url, - AliasExpDateSortOrder, PanelApiProbeTarget, SourcesYmlPatch, PANEL_API_DEFAULT_RETRY_AFTER_SECS, - PANEL_API_MAX_RETRY_AFTER_SECS, + build_panel_api_probe_targets, build_user_api_account_info_input_source, + panel_api_retry_after_from_header_value, panel_api_retryable_status, resolve_batch_alias_path, + PanelApiProbeTarget, PANEL_API_DEFAULT_RETRY_AFTER_SECS, PANEL_API_MAX_RETRY_AFTER_SECS, + }; + use crate::{ + api::source_yml_patch::{apply_sources_yml_patches, resolve_provisioned_account_base_url, SourcesYmlPatch}, + model::{ConfigInput, ConfigProvider}, + repository::AliasExpDateSortOrder, }; - use crate::model::{ConfigInput, ConfigProvider}; use axum::http::StatusCode; use shared::model::{ ConfigInputAliasDto, ConfigInputDto, ConfigProviderDto, InputType, ProviderUrlSelectionPolicy, SourcesConfigDto, @@ -4194,6 +3755,12 @@ mod tests { } } + #[test] + fn batch_alias_storage_never_falls_back_to_source_yml() { + assert!(resolve_batch_alias_path(None).expect("non-batch input").is_none()); + assert!(resolve_batch_alias_path(Some("provider://not-a-csv")).is_err()); + } + fn source_doc_with_aliases(aliases: Vec) -> SourcesConfigDto { SourcesConfigDto { inputs: vec![ConfigInputDto { @@ -4275,7 +3842,7 @@ mod tests { let changed = apply_sources_yml_patches( &mut doc, - &[SourcesYmlPatch::UpdateExpDate { + &[SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: Arc::from("cdn-dev"), account_name: Arc::from("cdn-dev"), exp_date: 20, @@ -4296,7 +3863,7 @@ mod tests { let changed = apply_sources_yml_patches( &mut doc, - &[SourcesYmlPatch::UpdateExpDate { + &[SourcesYmlPatch::UpdatePanelAccountExpiry { input_name: Arc::from("cdn-dev"), account_name: Arc::from("cdn-dev-old"), exp_date: 20, diff --git a/backend/app/src/api/source_yml_patch.rs b/backend/app/src/api/source_yml_patch.rs new file mode 100644 index 000000000..0cf7f55f8 --- /dev/null +++ b/backend/app/src/api/source_yml_patch.rs @@ -0,0 +1,1410 @@ +//! Central orchestration for targeted `source.yml` patches. +//! +//! This module owns the full patch transaction: acquire lock → read → semantic +//! apply → text patch → verify → backup/replace → mark internal revision. +//! Background tasks (expiry worker, Panel API) construct `SourcesYmlPatch` +//! commands and call [`execute_source_yml_patches`] — they never read, mutate, +//! or serialize the file themselves. + +use crate::{ + config_loader::{ + source_patch::{ + apply_scalar_edits, build_alias_addition_edit, build_alias_removal_edits, build_alias_sequence_edit, + build_alias_sort_edit, build_field_insertion_edit, find_input, parse_and_validate_patched_text, + parse_patch_document, serialize_yaml_scalar, span_byte_range, TextEdit, + }, + write_config_text_file, + }, + model::{is_input_expired, is_input_expired_at}, + repository::AliasExpDateSortOrder, +}; +use log::warn; +use shared::{ + error::TuliproxError, + model::{ConfigInputAliasDto, ConfigInputDto, SourcesConfigDto}, + utils::Internable, +}; +use std::{ + collections::{HashMap, HashSet}, + ops::Range, + path::Path, + sync::Arc, +}; +use tuliprox_core::model::AppConfig; +use url::Url; + +// --------------------------------------------------------------------------- +// Command enum +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub(crate) enum SourcesYmlPatch { + SetFetchedExpiry { + input_name: Arc, + account_name: Arc, + exp_date: i64, + disable: bool, + }, + UpdatePanelAccountExpiry { + input_name: Arc, + account_name: Arc, + exp_date: i64, + }, + UpdatePanelApiCredits { + input_name: Arc, + credits: String, + }, + SortAliases { + input_name: Arc, + order: AliasExpDateSortOrder, + }, + UpdateRootCredentials { + input_name: Arc, + username: String, + password: String, + exp_date: Option, + }, + PersistProvisionedAccount { + input_name: Arc, + username: String, + password: String, + exp_date: Option, + }, + UpdateAliasCredentials { + input_name: Arc, + alias_name: Arc, + username: String, + password: String, + exp_date: Option, + }, + AddAlias { + input_name: Arc, + alias_name: Arc, + base_url: String, + username: String, + password: String, + exp_date: Option, + }, + RemoveExpiredAliases { + input_name: Arc, + }, +} + +impl SourcesYmlPatch { + /// Structural alias edits change subsequent YAML byte spans and therefore + /// form their own planning step. Consecutive scalar-only commands can be + /// planned and validated together without reparsing the whole document. + const fn changes_alias_structure(&self) -> bool { + matches!( + self, + Self::SortAliases { .. } + | Self::PersistProvisionedAccount { .. } + | Self::AddAlias { .. } + | Self::RemoveExpiredAliases { .. } + ) + } +} + +// --------------------------------------------------------------------------- +// Semantic application (operates on the DTO clone) +// --------------------------------------------------------------------------- + +fn update_url_query_credentials_if_present(url: &mut String, username: &str, password: &str) { + let Ok(mut parsed) = Url::parse(url.as_str()) else { + return; + }; + let mut pairs: Vec<(String, String)> = parsed.query_pairs().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + let mut has_user = false; + let mut has_pass = false; + for (k, v) in &mut pairs { + if k.eq_ignore_ascii_case("username") { + *v = username.to_string(); + has_user = true; + } else if k.eq_ignore_ascii_case("password") { + *v = password.to_string(); + has_pass = true; + } + } + if has_user || has_pass { + if !has_user { + pairs.push(("username".to_string(), username.to_string())); + } + if !has_pass { + pairs.push(("password".to_string(), password.to_string())); + } + parsed.query_pairs_mut().clear(); + { + let mut qp = parsed.query_pairs_mut(); + for (k, v) in pairs { + qp.append_pair(k.as_str(), v.as_str()); + } + } + *url = parsed.to_string(); + } +} + +fn sort_aliases_by_exp_date_order(aliases: &mut [ConfigInputAliasDto], order: AliasExpDateSortOrder) -> bool { + if aliases.len() < 2 { + return false; + } + let compare = |a: &ConfigInputAliasDto, b: &ConfigInputAliasDto| { + crate::repository::compare_alias_exp_date_with_order(a, b, order) + }; + if aliases.windows(2).all(|pair| compare(&pair[0], &pair[1]) != std::cmp::Ordering::Greater) { + return false; + } + aliases.sort_by(compare); + true +} + +const MAX_ALIAS_NAME_ATTEMPTS: usize = 1000; + +pub(crate) fn derive_unique_alias_name(existing: &[Arc], input_name: &Arc, username: &str) -> Arc { + derive_unique_alias_name_with( + |candidate| existing.iter().any(|name| name.as_ref() == candidate), + input_name, + username, + ) + .intern() +} + +pub(crate) fn resolve_provisioned_account_base_url( + input_url: &str, + base_url_from_response: Option<&str>, + username: &str, + password: &str, +) -> String { + use shared::utils::{get_base_url_from_str, PROVIDER_SCHEME_PREFIX}; + if input_url.starts_with(PROVIDER_SCHEME_PREFIX) { + let mut provider_url = input_url.to_string(); + update_url_query_credentials_if_present(&mut provider_url, username, password); + return provider_url; + } + + let base_url = base_url_from_response.map_or_else(|| input_url.to_string(), ToString::to_string); + if let Some(origin) = get_base_url_from_str(base_url.as_str()) { + let trimmed_origin = origin.trim(); + if !trimmed_origin.is_empty() && !trimmed_origin.eq_ignore_ascii_case("null") { + return origin; + } + } + + let trimmed_base = base_url.trim(); + if !trimmed_base.is_empty() && !trimmed_base.eq_ignore_ascii_case("null") { + return base_url; + } + + input_url.to_string() +} + +fn append_sources_yml_alias( + input_name: &Arc, + input: &mut ConfigInputDto, + alias_name: Arc, + base_url: String, + username: String, + password: String, + exp_date: Option, +) -> Result { + let input_type = input.input_type; + let aliases = input.aliases.get_or_insert_with(Vec::new); + let next_index = aliases.iter().map(|alias| alias.id).max().unwrap_or(0); + if next_index == u16::MAX { + return Err(TuliproxError::ConfigPanelApi(format!( + "panel_api: cannot add alias for '{input_name}': alias id overflow" + ))); + } + + let mut alias = ConfigInputAliasDto { + id: 0, + name: alias_name, + url: base_url, + username: Some(username), + password: Some(password), + priority: 0, + max_connections: 1, + exp_date, + enabled: true, + stalker: None, + }; + alias.prepare(next_index, &input_type)?; + aliases.push(alias); + Ok(aliases.len().saturating_sub(1)) +} + +pub(crate) fn derive_unique_alias_name_set( + existing: &std::collections::HashSet>, + input_name: &Arc, + username: &str, +) -> String { + derive_unique_alias_name_with(|candidate| existing.contains(candidate), input_name, username) +} + +fn derive_unique_alias_name_with(mut contains: impl FnMut(&str) -> bool, input_name: &str, username: &str) -> String { + let base = format!("{input_name}-{username}"); + if !contains(base.as_str()) { + return base; + } + for i in 2..MAX_ALIAS_NAME_ATTEMPTS { + let cand = format!("{base}-{i}"); + if !contains(cand.as_str()) { + return cand; + } + } + warn!( + "derive_unique_alias_name: exhausted {MAX_ALIAS_NAME_ATTEMPTS} attempts for base '{base}'; returning potentially duplicate name" + ); + base +} + +#[allow(clippy::too_many_lines)] +pub(crate) fn apply_sources_yml_patches( + doc: &mut SourcesConfigDto, + patches: &[SourcesYmlPatch], +) -> Result { + if patches.is_empty() { + return Ok(false); + } + + let mut changed = false; + let mut inputs_by_name: HashMap, usize> = HashMap::with_capacity(doc.inputs.len()); + let mut alias_indices: Vec, usize>> = Vec::with_capacity(doc.inputs.len()); + for (idx, input) in doc.inputs.iter().enumerate() { + inputs_by_name.insert(input.name.clone(), idx); + let map = input + .aliases + .as_ref() + .map(|aliases| { + aliases + .iter() + .enumerate() + .map(|(idx, alias)| (alias.name.clone(), idx)) + .collect::, usize>>() + }) + .unwrap_or_default(); + alias_indices.push(map); + } + + for patch in patches { + match patch { + SourcesYmlPatch::SetFetchedExpiry { input_name, account_name, exp_date, disable } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let account_changed = + doc.inputs[idx].update_account_expiration_date(account_name.as_ref(), *exp_date, *disable)?; + if account_changed { + changed = true; + } + } + SourcesYmlPatch::UpdatePanelAccountExpiry { input_name, account_name, exp_date } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + if account_name == input_name { + if doc.inputs[idx].exp_date != Some(*exp_date) + || !doc.inputs[idx].enabled + || doc.inputs[idx].max_connections != 1 + { + doc.inputs[idx].exp_date = Some(*exp_date); + doc.inputs[idx].enabled = true; + doc.inputs[idx].max_connections = 1; + changed = true; + } + continue; + } + let Some(alias_idx) = alias_indices[idx].get(account_name).copied() else { + return Err(TuliproxError::ConfigPanelApi(format!( + "source.yml patch target alias '{account_name}' under input '{input_name}' was not found" + ))); + }; + let aliases = doc.inputs[idx].aliases.as_mut().ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch: input '{input_name}' has no aliases")) + })?; + if aliases[alias_idx].exp_date != Some(*exp_date) || aliases[alias_idx].max_connections != 1 { + aliases[alias_idx].exp_date = Some(*exp_date); + aliases[alias_idx].max_connections = 1; + changed = true; + } + } + SourcesYmlPatch::UpdatePanelApiCredits { input_name, credits } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let Some(panel_api) = doc.inputs[idx].panel_api.as_mut() else { + return Err(TuliproxError::ConfigPanelApi(format!( + "source.yml patch: could not find panel_api for input '{input_name}'" + ))); + }; + if panel_api.credits.as_deref().map(str::trim) != Some(credits.trim()) { + panel_api.credits = Some(credits.trim().to_string()); + changed = true; + } + } + SourcesYmlPatch::SortAliases { input_name, order } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { + continue; + }; + if sort_aliases_by_exp_date_order(aliases, *order) { + alias_indices[idx] = + aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); + changed = true; + } + } + SourcesYmlPatch::UpdateRootCredentials { input_name, username, password, exp_date } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let input = &mut doc.inputs[idx]; + let exp_date_changed = exp_date.is_some() && input.exp_date != *exp_date; + if input.username.as_deref() != Some(username.as_str()) + || input.password.as_deref() != Some(password.as_str()) + || exp_date_changed + { + input.username = Some(username.clone()); + input.password = Some(password.clone()); + input.enabled = true; + input.max_connections = 1; + if let Some(exp_date) = *exp_date { + input.exp_date = Some(exp_date); + } + update_url_query_credentials_if_present(&mut input.url, username, password); + changed = true; + } + } + SourcesYmlPatch::PersistProvisionedAccount { input_name, username, password, exp_date } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let input = &mut doc.inputs[idx]; + let current_root_is_usable = input.exp_date.is_some() + && !is_input_expired_at(input.exp_date, jsonwebtoken::get_current_timestamp()); + if current_root_is_usable { + let mut existing_names = vec![input.name.clone()]; + if let Some(aliases) = input.aliases.as_ref() { + existing_names.extend(aliases.iter().map(|alias| alias.name.clone())); + } + let alias_name = derive_unique_alias_name(&existing_names, &input.name, username); + let base_url = resolve_provisioned_account_base_url(input.url.as_str(), None, username, password); + let alias_idx = append_sources_yml_alias( + input_name, + input, + Arc::clone(&alias_name), + base_url, + username.clone(), + password.clone(), + *exp_date, + )?; + alias_indices[idx].insert(Arc::clone(&alias_name), alias_idx); + if let Some(aliases) = input.aliases.as_mut() { + if sort_aliases_by_exp_date_order(aliases, AliasExpDateSortOrder::NewestFirst) { + alias_indices[idx] = + aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); + } + } + changed = true; + continue; + } + let exp_date_changed = exp_date.is_some() && input.exp_date != *exp_date; + if input.username.as_deref() != Some(username.as_str()) + || input.password.as_deref() != Some(password.as_str()) + || exp_date_changed + { + input.username = Some(username.clone()); + input.password = Some(password.clone()); + input.enabled = true; + input.max_connections = 1; + if let Some(exp_date) = *exp_date { + input.exp_date = Some(exp_date); + } + update_url_query_credentials_if_present(&mut input.url, username, password); + changed = true; + } + } + SourcesYmlPatch::UpdateAliasCredentials { input_name, alias_name, username, password, exp_date } => { + let idx = *inputs_by_name.get(input_name.as_ref()).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let Some(alias_idx) = alias_indices[idx].get(alias_name).copied() else { + return Err(TuliproxError::ConfigPanelApi(format!( + "source.yml patch target alias '{alias_name}' under input '{input_name}' was not found" + ))); + }; + let aliases = doc.inputs[idx].aliases.as_mut().ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch: input '{input_name}' has no aliases")) + })?; + let alias = &mut aliases[alias_idx]; + let exp_date_changed = exp_date.is_some() && alias.exp_date != *exp_date; + if alias.username.as_deref() != Some(username.as_str()) + || alias.password.as_deref() != Some(password.as_str()) + || exp_date_changed + { + alias.username = Some(username.clone()); + alias.password = Some(password.clone()); + alias.max_connections = 1; + if let Some(exp_date) = *exp_date { + alias.exp_date = Some(exp_date); + } + update_url_query_credentials_if_present(&mut alias.url, username, password); + changed = true; + } + } + SourcesYmlPatch::AddAlias { input_name, alias_name, base_url, username, password, exp_date } => { + let idx = *inputs_by_name.get(input_name).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let alias_idx = append_sources_yml_alias( + input_name, + &mut doc.inputs[idx], + Arc::clone(alias_name), + base_url.clone(), + username.clone(), + password.clone(), + *exp_date, + )?; + alias_indices[idx].insert(Arc::clone(alias_name), alias_idx); + changed = true; + } + SourcesYmlPatch::RemoveExpiredAliases { input_name } => { + let idx = *inputs_by_name.get(input_name).ok_or_else(|| { + TuliproxError::ConfigPanelApi(format!("source.yml patch target input '{input_name}' was not found")) + })?; + let Some(aliases) = doc.inputs[idx].aliases.as_mut() else { + continue; + }; + let before = aliases.len(); + aliases.retain(|a| !is_input_expired(a.exp_date)); + if aliases.len() != before { + alias_indices[idx] = + aliases.iter().enumerate().map(|(idx, alias)| (alias.name.clone(), idx)).collect(); + changed = true; + } + } + } + } + + Ok(changed) +} + +// --------------------------------------------------------------------------- +// Text-edit planning (translates semantic diff into byte edits) +// --------------------------------------------------------------------------- + +fn plan_text_edits( + original_text: &str, + before: &SourcesConfigDto, + expected: &SourcesConfigDto, + _patches: &[SourcesYmlPatch], +) -> Result, TuliproxError> { + let doc = parse_patch_document(original_text)?; + let mut edits: Vec = Vec::new(); + + for (before_input, expected_input) in before.inputs.iter().zip(expected.inputs.iter()) { + let input_name = &expected_input.name; + + // Scalar field diffs on the root input + plan_scalar_field_edits(original_text, &doc, input_name, before_input, expected_input, &mut edits)?; + + // Alias structural changes + let before_aliases = before_input.aliases.as_deref().unwrap_or_default(); + let expected_aliases = expected_input.aliases.as_deref().unwrap_or_default(); + + if before_aliases.len() != expected_aliases.len() + || before_aliases.iter().zip(expected_aliases.iter()).any(|(a, b)| a.name != b.name) + { + // Structural change: aliases were added, removed, or reordered + plan_alias_structural_edits(original_text, &doc, input_name, before_aliases, expected_aliases, &mut edits)?; + } else { + // Same aliases in same order — check for scalar diffs within each alias + for (alias_idx, (before_alias, expected_alias)) in + before_aliases.iter().zip(expected_aliases.iter()).enumerate() + { + plan_alias_scalar_edits( + original_text, + &doc, + input_name, + alias_idx, + before_alias, + expected_alias, + &mut edits, + )?; + } + } + } + + Ok(edits) +} + +/// Byte span of an optional projected scalar. +fn opt_span(spanned: Option<&serde_saphyr::Spanned>) -> Result>, TuliproxError> { + spanned.map(span_byte_range).transpose() +} + +/// Plans a single scalar field change. +/// +/// An existing field is replaced in place through its exact span. A missing field is inserted +/// below the first anchor that exists, so key order stays deterministic and no other byte moves. +fn plan_field( + text: &str, + existing: Option>, + anchors: &[Option>], + key: &str, + value: &str, + edits: &mut Vec, +) -> Result<(), TuliproxError> { + if let Some(range) = existing { + edits.push(TextEdit { range, replacement: value.to_string() }); + return Ok(()); + } + + let anchor = anchors.iter().flatten().next().ok_or_else(|| { + TuliproxError::Config(format!("source.yml patch: no insertion anchor found for field '{key}'")) + })?; + edits.push(build_field_insertion_edit(text, anchor, key, value)?); + Ok(()) +} + +/// Plans the scalar edits shared by root inputs and aliases. +/// +/// `name_span` anchors every field that has no better sibling to attach to. +struct AccountFieldSpans { + name: Range, + enabled: Option>, + url: Option>, + username: Option>, + password: Option>, + exp_date: Option>, + max_connections: Option>, +} + +impl AccountFieldSpans { + fn from_input(value: &crate::config_loader::source_patch::PatchInput) -> Result { + Ok(Self { + name: span_byte_range(&value.name)?, + enabled: opt_span(value.enabled.as_ref())?, + url: opt_span(value.url.as_ref())?, + username: opt_span(value.username.as_ref())?, + password: opt_span(value.password.as_ref())?, + exp_date: opt_span(value.exp_date.as_ref())?, + max_connections: opt_span(value.max_connections.as_ref())?, + }) + } + + fn from_alias(value: &crate::config_loader::source_patch::PatchAlias) -> Result { + Ok(Self { + name: span_byte_range(&value.name)?, + enabled: opt_span(value.enabled.as_ref())?, + url: opt_span(value.url.as_ref())?, + username: opt_span(value.username.as_ref())?, + password: opt_span(value.password.as_ref())?, + exp_date: opt_span(value.exp_date.as_ref())?, + max_connections: opt_span(value.max_connections.as_ref())?, + }) + } + + fn name_anchor(&self) -> Range { self.name.clone() } +} + +/// Values that changed between `before` and `expected` for one account. +struct AccountFieldChanges<'a> { + enabled: Option, + url: Option<&'a str>, + username: Option<&'a str>, + password: Option<&'a str>, + exp_date: Option, + max_connections: Option, +} + +/// Plans every scalar edit of one account (root input or alias) from its spans and changes. +fn plan_account_scalar_edits( + text: &str, + spans: &AccountFieldSpans, + changes: &AccountFieldChanges<'_>, + edits: &mut Vec, +) -> Result<(), TuliproxError> { + if let Some(exp_date) = changes.exp_date { + let value = serialize_yaml_scalar(&exp_date)?; + let anchors = [spans.password.clone(), spans.username.clone(), Some(spans.name_anchor())]; + plan_field(text, spans.exp_date.clone(), &anchors, "exp_date", &value, edits)?; + } + + if let Some(enabled) = changes.enabled { + // A missing `enabled` already means `true`, so only the disabling case needs an insertion. + if spans.enabled.is_some() || !enabled { + let value = serialize_yaml_scalar(&enabled)?; + let anchors = [Some(spans.name_anchor())]; + plan_field(text, spans.enabled.clone(), &anchors, "enabled", &value, edits)?; + } + } + + if let Some(max_connections) = changes.max_connections { + // A missing `max_connections` already means `0`, so only a real limit needs an insertion. + if spans.max_connections.is_some() || max_connections != 0 { + let value = serialize_yaml_scalar(&max_connections)?; + let anchors = + [spans.exp_date.clone(), spans.password.clone(), spans.username.clone(), Some(spans.name_anchor())]; + plan_field(text, spans.max_connections.clone(), &anchors, "max_connections", &value, edits)?; + } + } + + if let Some(username) = changes.username { + let value = serialize_yaml_scalar(&username)?; + let anchors = [spans.url.clone(), Some(spans.name_anchor())]; + plan_field(text, spans.username.clone(), &anchors, "username", &value, edits)?; + } + + if let Some(password) = changes.password { + let value = serialize_yaml_scalar(&password)?; + let anchors = [spans.username.clone(), spans.url.clone(), Some(spans.name_anchor())]; + plan_field(text, spans.password.clone(), &anchors, "password", &value, edits)?; + } + + // `url` is mandatory in every account, so it is only ever replaced. + if let Some(url) = changes.url { + if let Some(range) = spans.url.clone() { + edits.push(TextEdit { range, replacement: serialize_yaml_scalar(&url)? }); + } + } + + Ok(()) +} + +fn plan_scalar_field_edits( + text: &str, + doc: &crate::config_loader::source_patch::SourcePatchDocument, + input_name: &Arc, + before: &ConfigInputDto, + expected: &ConfigInputDto, + edits: &mut Vec, +) -> Result<(), TuliproxError> { + let patch_input = find_input(doc, input_name.as_ref())?; + let value = &patch_input.value; + + let spans = AccountFieldSpans::from_input(value)?; + let changes = AccountFieldChanges { + enabled: (before.enabled != expected.enabled).then_some(expected.enabled), + url: (before.url != expected.url).then_some(expected.url.as_str()), + username: changed_credential(before.username.as_ref(), expected.username.as_ref()), + password: changed_credential(before.password.as_ref(), expected.password.as_ref()), + exp_date: (before.exp_date != expected.exp_date).then_some(expected.exp_date).flatten(), + max_connections: (before.max_connections != expected.max_connections).then_some(expected.max_connections), + }; + plan_account_scalar_edits(text, &spans, &changes, edits)?; + + plan_panel_api_credits_edit(text, value, before, expected, edits) +} + +/// Returns the new credential when it changed to a concrete value. +/// +/// Clearing a credential is not expressible as a scalar edit, so it is left to the semantic +/// verification step to reject such a patch instead of silently dropping the key. +fn changed_credential<'a>(before: Option<&String>, expected: Option<&'a String>) -> Option<&'a str> { + match expected { + Some(value) if before.map(String::as_str) != Some(value.as_str()) => Some(value.as_str()), + _ => None, + } +} + +fn plan_panel_api_credits_edit( + text: &str, + value: &crate::config_loader::source_patch::PatchInput, + before: &ConfigInputDto, + expected: &ConfigInputDto, + edits: &mut Vec, +) -> Result<(), TuliproxError> { + let (Some(before_panel), Some(expected_panel)) = (&before.panel_api, &expected.panel_api) else { + return Ok(()); + }; + if before_panel.credits == expected_panel.credits { + return Ok(()); + } + let Some(expected_credits) = &expected_panel.credits else { + return Ok(()); + }; + let Some(panel_spanned) = &value.panel_api else { + return Ok(()); + }; + + let new_val = serialize_yaml_scalar(expected_credits)?; + let existing = opt_span(panel_spanned.value.credits.as_ref())?; + // Without an existing `credits` key the first child of the `panel_api` mapping is the anchor. + let mapping_anchor = Some(span_byte_range(panel_spanned)?); + plan_field(text, existing, &[mapping_anchor], "credits", &new_val, edits) +} + +fn plan_alias_scalar_edits( + text: &str, + doc: &crate::config_loader::source_patch::SourcePatchDocument, + input_name: &Arc, + alias_idx: usize, + before: &ConfigInputAliasDto, + expected: &ConfigInputAliasDto, + edits: &mut Vec, +) -> Result<(), TuliproxError> { + let patch_input = find_input(doc, input_name.as_ref())?; + let Some(aliases_spanned) = &patch_input.value.aliases else { + return Ok(()); + }; + let Some(alias_spanned) = aliases_spanned.value.get(alias_idx) else { + return Ok(()); + }; + + let spans = AccountFieldSpans::from_alias(&alias_spanned.value)?; + let changes = AccountFieldChanges { + enabled: (before.enabled != expected.enabled).then_some(expected.enabled), + url: (before.url != expected.url).then_some(expected.url.as_str()), + username: changed_credential(before.username.as_ref(), expected.username.as_ref()), + password: changed_credential(before.password.as_ref(), expected.password.as_ref()), + exp_date: (before.exp_date != expected.exp_date).then_some(expected.exp_date).flatten(), + max_connections: (before.max_connections != expected.max_connections).then_some(expected.max_connections), + }; + plan_account_scalar_edits(text, &spans, &changes, edits) +} + +fn plan_alias_structural_edits( + text: &str, + doc: &crate::config_loader::source_patch::SourcePatchDocument, + input_name: &Arc, + before_aliases: &[ConfigInputAliasDto], + expected_aliases: &[ConfigInputAliasDto], + edits: &mut Vec, +) -> Result<(), TuliproxError> { + let before_names: Vec<&str> = before_aliases.iter().map(|a| a.name.as_ref()).collect(); + let expected_names: Vec<&str> = expected_aliases.iter().map(|a| a.name.as_ref()).collect(); + let before_set: HashSet<&str> = before_names.iter().copied().collect(); + let expected_set: HashSet<&str> = expected_names.iter().copied().collect(); + + let expected_existing_order: Vec<&str> = + expected_names.iter().copied().filter(|name| before_set.contains(name)).collect(); + let surviving_before_order: Vec<&str> = + before_names.iter().copied().filter(|name| expected_set.contains(name)).collect(); + let added: Vec<&ConfigInputAliasDto> = + expected_aliases.iter().filter(|alias| !before_set.contains(alias.name.as_ref())).collect(); + let appended_without_reordering = expected_existing_order == surviving_before_order + && expected_names + .get(expected_names.len().saturating_sub(added.len())..) + .is_some_and(|suffix| suffix.iter().copied().eq(added.iter().map(|alias| alias.name.as_ref()))); + + // A command such as PersistProvisionedAccount can add and sort in one semantic operation. + // Rebuild the sequence from opaque existing blocks so the final order is represented by one + // non-overlapping edit. Sequential command batches use this same path one command at a time. + if !appended_without_reordering { + if let Some(edit) = build_alias_sequence_edit(text, doc, input_name.as_ref(), expected_aliases)? { + edits.push(edit); + return Ok(()); + } + } + + // Removed aliases + let removed: Vec<&str> = before_names.iter().filter(|name| !expected_set.contains(**name)).copied().collect(); + if !removed.is_empty() { + let removal_edits = build_alias_removal_edits(text, doc, input_name.as_ref(), &removed)?; + edits.extend(removal_edits); + } + + // Added aliases + for expected_alias in added { + let add_edit = build_alias_addition_edit(text, doc, input_name.as_ref(), expected_alias)?; + edits.push(add_edit); + } + + // Sort: if same set of names but different order + if removed.is_empty() && before_aliases.len() == expected_aliases.len() { + let mut sorted_before = before_names.clone(); + sorted_before.sort_unstable(); + let mut sorted_expected = expected_names.clone(); + sorted_expected.sort_unstable(); + if sorted_before == sorted_expected && before_names != expected_names { + if let Some(sort_edit) = build_alias_sort_edit(text, doc, input_name.as_ref(), &expected_names)? { + edits.push(sort_edit); + } + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Central transaction +// --------------------------------------------------------------------------- + +fn apply_patch_planning_step( + expected: &mut SourcesConfigDto, + patched_text: &mut String, + patches: &[SourcesYmlPatch], +) -> Result { + let mut next_expected = expected.clone(); + if !apply_sources_yml_patches(&mut next_expected, patches)? { + return Ok(false); + } + let text_edits = plan_text_edits(patched_text, expected, &next_expected, patches)?; + let next_text = apply_scalar_edits(patched_text, text_edits)?; + parse_and_validate_patched_text(&next_text, &next_expected)?; + *expected = next_expected; + *patched_text = next_text; + Ok(true) +} + +/// Executes a batch of `SourcesYmlPatch` commands as a single atomic transaction. +/// +/// Returns `Ok(true)` if the file was written, `Ok(false)` if no change was needed. +/// The write lock is acquired internally — callers must NOT hold it. +pub(crate) async fn execute_source_yml_patches( + app_config: &Arc, + sources_path: &Path, + patches: &[SourcesYmlPatch], +) -> Result { + if patches.is_empty() { + return Ok(false); + } + + let _lock = app_config.file_locks.write_lock(sources_path).await; + + // Step 2: Read original bytes + let original_bytes = tokio::fs::read(sources_path) + .await + .map_err(|err| TuliproxError::ConfigPanelApi(format!("source.yml patch: failed to read file: {err}")))?; + // Step 3: Compute revision + let original_revision = blake3::hash(&original_bytes); + let original_text = String::from_utf8(original_bytes) + .map_err(|_| TuliproxError::ConfigPanelApi("source.yml patch: file is not valid UTF-8".to_string()))?; + + // Step 4: Parse as SourcesConfigDto + let before: SourcesConfigDto = serde_saphyr::from_str(&original_text) + .map_err(|err| TuliproxError::ConfigPanelApi(format!("source.yml patch: failed to parse source.yml: {err}")))?; + + // Steps 5-9: Batch consecutive scalar commands into one clone/parse/validation pass. + // Structural alias commands remain sequential because they change the byte spans used by + // subsequent edits. The complete command list still produces one final atomic disk write. + let mut expected = before; + let mut patched_text = original_text; + let mut changed = false; + let mut patch_index = 0; + while patch_index < patches.len() { + let step_end = if patches[patch_index].changes_alias_structure() { + patch_index + 1 + } else { + patches[patch_index..] + .iter() + .position(SourcesYmlPatch::changes_alias_structure) + .map_or(patches.len(), |offset| patch_index + offset) + }; + changed |= apply_patch_planning_step(&mut expected, &mut patched_text, &patches[patch_index..step_end])?; + patch_index = step_end; + } + + // Step 6: No-op check + if !changed { + return Ok(false); + } + + // Step 10: Verify bytes outside edit ranges came from original + // (implicitly guaranteed by the edit model — apply_scalar_edits only modifies declared ranges) + + // Step 11: Re-read and check revision + let backup_dir = app_config.config.load().get_backup_dir().to_string(); + + // Step 12-13: Write with backup and atomic replace (includes revision check) + let written = write_config_text_file( + sources_path.to_string_lossy().as_ref(), + &backup_dir, + &patched_text, + "source.yml", + Some(original_revision), + ) + .await?; + + if written { + // Step 14: Mark internal write revision + app_config + .file_locks + .mark_internal_write_revision(sources_path) + .await + .map_err(|err| TuliproxError::Io(format!("Failed to track internal source update: {err}")))?; + } + + Ok(written) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config_loader::source_patch::{ensure_block_style_for_insertion, TextEdit}; + use shared::model::PanelApiConfigDto; + + fn edits_to_text(text: &str, edits: Vec) -> String { + crate::config_loader::source_patch::apply_scalar_edits(text, edits).expect("apply") + } + + fn find_edit<'a>(edits: &'a [TextEdit], needle: &str) -> Option<&'a TextEdit> { + edits.iter().find(|e| e.replacement.contains(needle)) + } + + #[test] + fn insertion_does_not_swallow_the_next_line() { + // Regression: the old code inserted after `line_end_offset`, which pointed past the newline, + // so the replacement concatenated into the following line. With a non-empty line after the + // anchor, the bug surfaces as `pass` with no separator. + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " username: user\n", + " password: pass\n", + " aliases:\n", + " - name: alias-a\n", + " url: http://a.example\n", + ); + let doc: crate::config_loader::source_patch::SourcePatchDocument = + serde_saphyr::from_str(fixture).expect("parse"); + + let before = ConfigInputDto { + name: "provider".into(), + url: "http://main.example".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + ..Default::default() + }; + let mut expected = before.clone(); + expected.exp_date = Some(1_700_000_000); + + let mut edits = Vec::new(); + plan_scalar_field_edits(fixture, &doc, &Arc::from("provider"), &before, &expected, &mut edits).expect("plan"); + + let patched = edits_to_text(fixture, edits); + for line in fixture.lines() { + assert!(patched.contains(line), "line lost after patch: {line}"); + } + assert!(patched.contains("exp_date: 1700000000")); + // The exact bug signature: `pass\n exp_date: ...` glued with no separator. + assert!( + !patched.contains("pass1700000000") && !patched.contains("pass exp_date"), + "next line must not be glued to the inserted value" + ); + } + + #[test] + fn enabling_then_disabling_yields_two_scalar_edits_without_insertions() { + // `enabled` is missing on the alias below; the patcher must replace or insert, never duplicate. + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " - name: alias-a\n", + " url: http://a.example\n", + ); + let doc: crate::config_loader::source_patch::SourcePatchDocument = + serde_saphyr::from_str(fixture).expect("parse"); + + let before = ConfigInputAliasDto { + name: "alias-a".into(), + url: "http://a.example".to_string(), + enabled: true, + ..Default::default() + }; + let mut expected = before.clone(); + expected.enabled = false; + + let mut edits = Vec::new(); + plan_alias_scalar_edits(fixture, &doc, &Arc::from("provider"), 0, &before, &expected, &mut edits) + .expect("plan"); + + assert_eq!(edits.len(), 1, "exactly one replacement edit, no insertion"); + let patched = edits_to_text(fixture, edits); + assert!(patched.contains("enabled: false")); + assert_eq!(patched.matches("enabled:").count(), 1, "no duplicate `enabled:` key"); + } + + #[test] + fn root_panel_credits_inserts_into_block_mapping() { + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " panel_api:\n", + " url: http://panel.example\n", + ); + let doc: crate::config_loader::source_patch::SourcePatchDocument = + serde_saphyr::from_str(fixture).expect("parse"); + + let before = ConfigInputDto { + name: "provider".into(), + url: "http://main.example".to_string(), + panel_api: Some(PanelApiConfigDto { url: "http://panel.example".to_string(), ..Default::default() }), + ..Default::default() + }; + let mut expected = before.clone(); + expected.panel_api.as_mut().expect("panel").credits = Some("42".to_string()); + + let mut edits = Vec::new(); + plan_scalar_field_edits(fixture, &doc, &Arc::from("provider"), &before, &expected, &mut edits).expect("plan"); + + assert!(find_edit(&edits, "credits").is_some(), "credits edit planned"); + let patched = edits_to_text(fixture, edits); + let reparsed: crate::config_loader::source_patch::SourcePatchDocument = + serde_saphyr::from_str(&patched).expect("reparse"); + let panel = reparsed.inputs[0].value.panel_api.as_ref().expect("panel"); + assert_eq!(panel.value.credits.as_ref().expect("credits").value, "42"); + } + + #[test] + fn flow_style_insertion_is_rejected_with_actionable_error() { + // `find_input` is the entry point; it does not reject flow-style itself. The rejector is + // `ensure_block_style_for_insertion`. We drive it directly to keep the unit test focused. + let fixture = "inputs:\n - { name: provider, url: http://main.example }\n"; + let doc: crate::config_loader::source_patch::SourcePatchDocument = + serde_saphyr::from_str(fixture).expect("parse"); + let input = &doc.inputs[0].value; + let anchor = span_byte_range(&input.name).expect("name span"); + + let err = ensure_block_style_for_insertion(fixture, &anchor, "exp_date").expect_err("must reject"); + let msg = format!("{err}"); + assert!( + msg.contains("cannot insert optional field 'exp_date' into a flow-style YAML mapping"), + "unexpected error: {msg}" + ); + } + + // ----------------------------------------------------------------------- + // End-to-end: the central transaction against a real on-disk source.yml. + // ----------------------------------------------------------------------- + + mod e2e { + use super::*; + use arc_swap::ArcSwap; + use shared::{ + model::{ConfigPaths, InputType, SourcesConfigDto}, + utils::Internable, + }; + use std::time::{SystemTime, UNIX_EPOCH}; + use tuliprox_core::{ + model::{Config, ConfigInput, MediaToolCapabilities, SourcesConfig}, + utils::FileLockManager, + }; + use tuliprox_repository::AliasExpDateSortOrder; + + fn build_app_config(backup_dir: &std::path::Path) -> Arc { + let input = ConfigInput { + id: 1, + name: "provider".intern(), + input_type: InputType::Xtream, + url: "http://main.example".to_string(), + username: Some("user".to_string()), + password: Some("pass".to_string()), + enabled: true, + priority: 0, + max_connections: 0, + aliases: None, + ..ConfigInput::default() + }; + let sources = SourcesConfig { inputs: vec![Arc::new(input)], ..SourcesConfig::default() }; + Arc::new(AppConfig { + config: Arc::new(ArcSwap::from_pointee(Config { + backup_dir: Some(backup_dir.to_string_lossy().into_owned()), + ..Config::default() + })), + sources: Arc::new(ArcSwap::from_pointee(sources)), + hdhomerun: Arc::new(arc_swap::ArcSwapOption::default()), + api_proxy: Arc::new(arc_swap::ArcSwapOption::default()), + file_locks: Arc::new(FileLockManager::default()), + paths: Arc::new(ArcSwap::from_pointee(ConfigPaths { + home_path: String::new(), + config_path: String::new(), + storage_path: String::new(), + config_file_path: String::new(), + sources_file_path: String::new(), + mapping_file_path: None, + mapping_files_used: None, + template_file_path: None, + template_files_used: None, + api_proxy_file_path: String::new(), + custom_stream_response_path: None, + })), + custom_stream_response: Arc::new(arc_swap::ArcSwapOption::default()), + access_token_secret: [0; 32], + encrypt_secret: [0; 16], + media_tools: Arc::new(MediaToolCapabilities::new()), + }) + } + + const FIXTURE: &str = "\ +inputs: + - name: provider + enabled: true + url: http://main.example + username: user + password: pass + aliases: + - name: alias-a + url: http://a.example + username: a-user + password: a-pass +sources: [] +"; + + fn unique_path(name: &str) -> std::path::PathBuf { + let nanos = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + std::env::temp_dir().join(format!("tuliprox-source-yml-patch-{nanos}-{name}")) + } + + #[tokio::test] + async fn end_to_end_patch_inserts_exp_date_and_writes_backup() { + let dir = unique_path("insert"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + tokio::fs::write(&source_path, FIXTURE).await.expect("write fixture"); + + let app_cfg = build_app_config(&backup_dir); + let name: Arc = Arc::from("provider"); + + let patches = [SourcesYmlPatch::SetFetchedExpiry { + input_name: name.clone(), + account_name: name.clone(), + exp_date: 1_700_000_000, + disable: false, + }]; + + let written = execute_source_yml_patches(&app_cfg, &source_path, &patches).await.expect("patch"); + assert!(written, "patch should report a write happened"); + + let patched_text = tokio::fs::read_to_string(&source_path).await.expect("read patched"); + assert!(patched_text.contains("exp_date: 1700000000"), "patched file must contain inserted exp_date"); + for line in FIXTURE.lines() { + assert!(patched_text.contains(line), "line lost after patch: {line}"); + } + + let backup_entries: Vec<_> = std::fs::read_dir(&backup_dir).expect("readdir").flatten().collect(); + assert!(!backup_entries.is_empty(), "backup directory must contain a backup of the original"); + let backup_path = backup_entries[0].path(); + let backup_text = tokio::fs::read_to_string(&backup_path).await.expect("read backup"); + assert_eq!(backup_text, FIXTURE, "backup must equal the original fixture byte-for-byte"); + + let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched_text).expect("reparse"); + let updated = parsed.inputs.iter().find(|i| i.name.as_ref() == "provider").expect("input"); + assert_eq!(updated.exp_date, Some(1_700_000_000)); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn scalar_patch_batch_updates_root_and_alias_in_one_planning_step() { + let dir = unique_path("scalar-batch"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + tokio::fs::write(&source_path, FIXTURE).await.expect("write fixture"); + + let app_cfg = build_app_config(&backup_dir); + let patches = [ + SourcesYmlPatch::SetFetchedExpiry { + input_name: Arc::from("provider"), + account_name: Arc::from("provider"), + exp_date: 1_700_000_000, + disable: false, + }, + SourcesYmlPatch::SetFetchedExpiry { + input_name: Arc::from("provider"), + account_name: Arc::from("alias-a"), + exp_date: 1_800_000_000, + disable: false, + }, + ]; + + assert!(execute_source_yml_patches(&app_cfg, &source_path, &patches).await.expect("patch")); + let patched = tokio::fs::read_to_string(&source_path).await.expect("read patched"); + let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched).expect("reparse"); + assert_eq!(parsed.inputs[0].exp_date, Some(1_700_000_000)); + assert_eq!(parsed.inputs[0].aliases.as_ref().expect("aliases")[0].exp_date, Some(1_800_000_000)); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn duplicate_alias_names_make_patch_fail_without_corrupting_file() { + let dir = unique_path("dup"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + + let fixture = "\ +inputs: + - name: provider + url: http://main.example + aliases: + - name: twin + url: http://first.example + - name: twin + url: http://second.example +sources: [] +"; + tokio::fs::write(&source_path, fixture).await.expect("write fixture"); + + let app_cfg = build_app_config(&backup_dir); + let name: Arc = Arc::from("provider"); + + let patches = [SourcesYmlPatch::SetFetchedExpiry { + input_name: name, + account_name: Arc::from("twin"), + exp_date: 1_700_000_000, + disable: false, + }]; + + let err = execute_source_yml_patches(&app_cfg, &source_path, &patches) + .await + .expect_err("duplicate alias must be rejected"); + let msg = format!("{err:?}"); + assert!(msg.contains("twin"), "error must name the duplicated alias, got: {msg}"); + + let after = tokio::fs::read_to_string(&source_path).await.expect("read after"); + assert_eq!(after, fixture, "failed patch must not modify the file"); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn unchanged_payload_is_a_no_op() { + let dir = unique_path("noop"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + tokio::fs::write(&source_path, FIXTURE).await.expect("write fixture"); + + let app_cfg = build_app_config(&backup_dir); + let name: Arc = Arc::from("provider"); + + let patches = + [SourcesYmlPatch::SortAliases { input_name: name, order: AliasExpDateSortOrder::NewestFirst }]; + + let written = execute_source_yml_patches(&app_cfg, &source_path, &patches).await.expect("noop"); + assert!(!written, "no-op patch must not report a write"); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn add_and_sort_alias_is_one_lossless_transaction() { + let dir = unique_path("add-sort"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + let fixture = concat!( + "inputs:\r\n", + " - name: provider\r\n", + " url: http://main.example\r\n", + " aliases:\r\n", + " # old account\r\n", + " - name: old\r\n", + " url: http://old.example\r\n", + " exp_date: 100\r\n", + "sources: []\r\n", + ); + tokio::fs::write(&source_path, fixture).await.expect("write fixture"); + let app_cfg = build_app_config(&backup_dir); + let patches = [ + SourcesYmlPatch::AddAlias { + input_name: Arc::from("provider"), + alias_name: Arc::from("new"), + base_url: "http://new.example".to_string(), + username: "new-user".to_string(), + password: "new-pass".to_string(), + exp_date: Some(300), + }, + SourcesYmlPatch::SortAliases { + input_name: Arc::from("provider"), + order: AliasExpDateSortOrder::NewestFirst, + }, + ]; + + assert!(execute_source_yml_patches(&app_cfg, &source_path, &patches).await.expect("patch")); + let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); + assert!(patched.find("name: new").expect("new") < patched.find("name: old").expect("old")); + assert_eq!(patched.matches("# old account").count(), 1); + assert!(patched.contains("\r\n")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn scalar_update_and_sort_aliases_are_both_persisted() { + let dir = unique_path("update-sort"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " - name: first\n", + " url: http://first.example\n", + " exp_date: 100\n", + " - name: second\n", + " url: http://second.example\n", + " exp_date: 200\n", + "sources: []\n", + ); + tokio::fs::write(&source_path, fixture).await.expect("write fixture"); + let app_cfg = build_app_config(&backup_dir); + let patches = [ + SourcesYmlPatch::UpdatePanelAccountExpiry { + input_name: Arc::from("provider"), + account_name: Arc::from("first"), + exp_date: 300, + }, + SourcesYmlPatch::SortAliases { + input_name: Arc::from("provider"), + order: AliasExpDateSortOrder::NewestFirst, + }, + ]; + + assert!(execute_source_yml_patches(&app_cfg, &source_path, &patches).await.expect("patch")); + let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); + assert!(patched.find("name: first").expect("first") < patched.find("name: second").expect("second")); + assert!(patched.contains("exp_date: 300")); + assert!(patched.contains("max_connections: 1")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn removing_all_aliases_is_semantically_valid() { + let dir = unique_path("remove-all"); + std::fs::create_dir_all(&dir).expect("mkdir"); + let source_path = dir.join("source.yml"); + let backup_dir = dir.join("backup"); + std::fs::create_dir_all(&backup_dir).expect("mkdir backup"); + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " - name: expired\n", + " url: http://expired.example\n", + " exp_date: 1\n", + "sources: []\n", + ); + tokio::fs::write(&source_path, fixture).await.expect("write fixture"); + let app_cfg = build_app_config(&backup_dir); + let patches = [SourcesYmlPatch::RemoveExpiredAliases { input_name: Arc::from("provider") }]; + + assert!(execute_source_yml_patches(&app_cfg, &source_path, &patches).await.expect("patch")); + let patched = tokio::fs::read_to_string(&source_path).await.expect("read"); + assert!(!patched.contains("name: expired")); + let parsed: SourcesConfigDto = serde_saphyr::from_str(&patched).expect("reparse"); + assert!(parsed.inputs[0].aliases.as_ref().is_none_or(Vec::is_empty)); + + let _ = std::fs::remove_dir_all(&dir); + } + } +} diff --git a/backend/app/src/api/tasks/xtream_expiry.rs b/backend/app/src/api/tasks/xtream_expiry.rs index 6479f0084..76c1a0d03 100644 --- a/backend/app/src/api/tasks/xtream_expiry.rs +++ b/backend/app/src/api/tasks/xtream_expiry.rs @@ -1,6 +1,9 @@ use crate::{ - api::{config_file::ConfigFile, model::AppState}, - config_loader::{persist_source_config_preserving_templates, read_sources_file_from_path}, + api::{ + config_file::ConfigFile, + model::AppState, + source_yml_patch::{execute_source_yml_patches, SourcesYmlPatch}, + }, iptv::xtream::get_xtream_stream_url_base, repository::{csv_patch_batch_update_exp_dates, get_csv_file_path, BatchExpDateUpdate}, utils::request, @@ -413,24 +416,23 @@ async fn persist_updates(app_state: &Arc, updates: &[(&Account, i64)]) let sources_path = app_state.app_config.paths.load().sources_file_path.clone(); let sources_path = std::path::Path::new(&sources_path); if !source_updates.is_empty() { - let _sources_lock = app_state.app_config.file_locks.write_lock(sources_path).await; - let mut sources = read_sources_file_from_path(sources_path, false, false, None).await?; - let mut source_changed = false; - for (key, input_name, account_name, exp_date) in source_updates { - if let Some(input) = sources.inputs.iter_mut().find(|input| input.name == input_name) { - source_changed |= - input.update_account_expiration_date(&account_name, exp_date, is_expired_at(exp_date, now))?; - updated_accounts.push(key); + let patches: Vec = source_updates + .iter() + .map(|(_, input_name, account_name, exp_date)| SourcesYmlPatch::SetFetchedExpiry { + input_name: Arc::clone(input_name), + account_name: Arc::clone(account_name), + exp_date: *exp_date, + disable: is_expired_at(*exp_date, now), + }) + .collect(); + match execute_source_yml_patches(&app_state.app_config, sources_path, &patches).await { + Ok(_) => { + updated_accounts.extend(source_updates.iter().map(|(key, _, _, _)| key.clone())); + } + Err(err) => { + // Failed patching leaves pending updates available for retry. + warn!("source.yml expiry patch failed, will retry: {err}"); } - } - if source_changed { - persist_source_config_preserving_templates(&app_state.app_config, Some(sources_path), sources).await?; - app_state - .app_config - .file_locks - .mark_internal_write_revision(sources_path) - .await - .map_err(|err| TuliproxError::Io(format!("Failed to track internal source update: {err}")))?; } } let mut persistence_error = None; @@ -698,6 +700,105 @@ mod tests { Ok(()) } + #[test] + fn fetched_future_expiry_updates_only_exp_date_on_root() -> Result<(), Box> { + let mut input = ConfigInputDto { + name: Arc::from("input"), + enabled: true, + max_connections: 5, + exp_date: Some(100), + ..Default::default() + }; + + assert!(input.update_account_expiration_date("input", 200, false)?); + assert_eq!(input.exp_date, Some(200)); + assert!(input.enabled); + assert_eq!(input.max_connections, 5); + Ok(()) + } + + #[test] + fn fetched_future_expiry_does_not_reenable_manually_disabled_root() -> Result<(), Box> { + let mut input = + ConfigInputDto { name: Arc::from("input"), enabled: false, exp_date: Some(100), ..Default::default() }; + + assert!(input.update_account_expiration_date("input", 200, false)?); + assert_eq!(input.exp_date, Some(200)); + assert!(!input.enabled); + Ok(()) + } + + #[test] + fn fetched_future_expiry_updates_only_exp_date_on_alias() -> Result<(), Box> { + let mut input = ConfigInputDto { + name: Arc::from("input"), + aliases: Some(vec![ConfigInputAliasDto { + name: Arc::from("alias"), + enabled: true, + max_connections: 3, + exp_date: Some(100), + ..Default::default() + }]), + ..Default::default() + }; + + assert!(input.update_account_expiration_date("alias", 200, false)?); + let alias = &input.aliases.as_ref().expect("aliases")[0]; + assert_eq!(alias.exp_date, Some(200)); + assert!(alias.enabled); + assert_eq!(alias.max_connections, 3); + Ok(()) + } + + #[test] + fn fetched_future_expiry_does_not_reenable_manually_disabled_alias() -> Result<(), Box> { + let mut input = ConfigInputDto { + name: Arc::from("input"), + aliases: Some(vec![ConfigInputAliasDto { + name: Arc::from("alias"), + enabled: false, + exp_date: Some(100), + ..Default::default() + }]), + ..Default::default() + }; + + assert!(input.update_account_expiration_date("alias", 200, false)?); + let alias = &input.aliases.as_ref().expect("aliases")[0]; + assert_eq!(alias.exp_date, Some(200)); + assert!(!alias.enabled); + Ok(()) + } + + #[test] + fn fetched_expired_expiry_disables_alias() -> Result<(), Box> { + let mut input = ConfigInputDto { + name: Arc::from("input"), + aliases: Some(vec![ConfigInputAliasDto { + name: Arc::from("alias"), + enabled: true, + exp_date: Some(100), + ..Default::default() + }]), + ..Default::default() + }; + + assert!(input.update_account_expiration_date("alias", 50, true)?); + let alias = &input.aliases.as_ref().expect("aliases")[0]; + assert_eq!(alias.exp_date, Some(50)); + assert!(!alias.enabled); + Ok(()) + } + + #[test] + fn unchanged_expiry_reports_no_change() -> Result<(), Box> { + let mut input = + ConfigInputDto { name: Arc::from("input"), enabled: true, exp_date: Some(100), ..Default::default() }; + + assert!(!input.update_account_expiration_date("input", 100, false)?); + Ok(()) + } + #[tokio::test] async fn expiry_state_is_atomically_replaced() -> Result<(), Box> { let dir = tempfile::tempdir()?; diff --git a/backend/config-loader/Cargo.toml b/backend/config-loader/Cargo.toml index 582d068b6..828b61a27 100644 --- a/backend/config-loader/Cargo.toml +++ b/backend/config-loader/Cargo.toml @@ -13,6 +13,7 @@ shared = { workspace = true } tuliprox-core = { workspace = true } tuliprox-repository = { workspace = true } arc-swap.workspace = true +blake3.workspace = true chrono.workspace = true log.workspace = true serde.workspace = true diff --git a/backend/config-loader/src/lib.rs b/backend/config-loader/src/lib.rs index 1c1de7185..ad6ba02d6 100644 --- a/backend/config-loader/src/lib.rs +++ b/backend/config-loader/src/lib.rs @@ -1,4 +1,5 @@ pub mod runtime_config_report; +pub mod source_patch; use arc_swap::{ArcSwap, ArcSwapAny}; use chrono::Local; @@ -667,24 +668,64 @@ async fn write_config_file( where T: ?Sized + Serialize, { - let path = PathBuf::from(file_path); - let filename = path.file_name().map_or(default_name.to_string(), |f| f.to_string_lossy().to_string()); - let mut serialized = String::new(); let options = serde_saphyr::ser_options! {prefer_block_scalars: false}; serde_saphyr::to_fmt_writer_with_options(&mut serialized, &config, options) .map_err(|err| TuliproxError::Config(format!("Could not serialize config: {err}")))?; - if file_exists_async(&path).await { - if let Ok(existing) = fs::read_to_string(&path).await { - if existing == serialized { - // info!("File {} unchanged, skipping write", path.display()); - return Ok(()); - } - } - } + write_config_text_file(file_path, backup_dir, &serialized, default_name, None).await?; + Ok(()) +} - if file_exists_async(&path).await { +/// Atomically writes raw text content to a config file with backup. +/// +/// Returns `Ok(true)` when the file was written, `Ok(false)` when the content was +/// already identical to the existing file (no-op). +/// +/// When `expected_revision` is provided, the destination file is re-read and its +/// blake3 hash is compared before writing. A mismatch produces a conflict error +/// and the file is left untouched. +pub async fn write_config_text_file( + file_path: &str, + backup_dir: &str, + content: &str, + default_name: &str, + expected_revision: Option, +) -> Result { + let path = PathBuf::from(file_path); + let filename = path.file_name().map_or(default_name.to_string(), |f| f.to_string_lossy().to_string()); + + let revision_content = if let Some(expected) = expected_revision { + let current = fs::read(&path).await.map_err(|err| { + TuliproxError::Config(format!( + "Could not re-read {} before applying an internal patch: {err}", + path.to_str().unwrap_or("?") + )) + })?; + if blake3::hash(¤t) != expected { + return Err(TuliproxError::Config( + "source.yml changed while an internal patch was being prepared; retrying later".to_string(), + )); + } + Some(current) + } else { + None + }; + + let destination_exists = if let Some(current) = revision_content.as_deref() { + if current == content.as_bytes() { + return Ok(false); + } + true + } else { + let exists = file_exists_async(&path).await; + if exists && fs::read_to_string(&path).await.is_ok_and(|existing| existing == content) { + return Ok(false); + } + exists + }; + + if destination_exists { fs::create_dir_all(backup_dir) .await .map_err(|err| TuliproxError::Config(format!("Could not create backup directory {backup_dir}: {err}")))?; @@ -713,18 +754,22 @@ where Local::now().timestamp_nanos_opt().unwrap_or_default() )); - fs::write(&tmp_path, serialized).await.map_err(|err| { - TuliproxError::Config(format!("Could not write temp file {}: {err}", tmp_path.to_str().unwrap_or("?"))) - })?; + if let Err(err) = fs::write(&tmp_path, content).await { + let _ = fs::remove_file(&tmp_path).await; + return Err(TuliproxError::Config(format!( + "Could not write temp file {}: {err}", + tmp_path.to_str().unwrap_or("?") + ))); + } match fs::rename(&tmp_path, &path).await { - Ok(()) => Ok(()), + Ok(()) => Ok(true), Err(err) => { // Windows doesn't allow overwriting an existing file via rename. #[cfg(windows)] { if replace_file_windows(&tmp_path, &path).is_ok() { - return Ok(()); + return Ok(true); } } @@ -870,7 +915,7 @@ pub async fn save_main_config(file_path: &str, backup_dir: &str, config: &Config write_config_file(file_path, backup_dir, config, "config.yml").await } -pub async fn save_sources_config(file_path: &str, backup_dir: &str, config: &T) -> Result<(), TuliproxError> +async fn save_sources_config(file_path: &str, backup_dir: &str, config: &T) -> Result<(), TuliproxError> where T: ?Sized + Serialize, { @@ -956,7 +1001,14 @@ pub async fn persist_templates_config( save_templates_config(&template_file, config.get_backup_dir().as_ref(), template_definition).await } -pub async fn persist_source_config( +/// Canonicalizes and rewrites the entire `source.yml` document. +/// +/// This performs a whole-file YAML serialization: comments, blank lines, key +/// order and quoting are normalized away. It must only be used for an explicit +/// user edit (Source Editor save) or a controlled migration — never by +/// background tasks. Background mutations must go through the targeted +/// lossless patch pipeline (`api::source_yml_patch::execute_source_yml_patches`). +pub async fn replace_source_config_from_user_edit( app_config: &Arc, source_file_path: Option<&Path>, doc: SourcesConfigDto, @@ -965,15 +1017,6 @@ pub async fn persist_source_config( persist_sanitized_source_config(app_config, source_file_path, doc, &source_config).await } -pub async fn persist_source_config_preserving_templates( - app_config: &Arc, - source_file_path: Option<&Path>, - doc: SourcesConfigDto, -) -> Result { - let source_config = sanitize_sources_for_persist_preserving_templates(doc.clone()).await; - persist_sanitized_source_config(app_config, source_file_path, doc, &source_config).await -} - async fn persist_sanitized_source_config( app_config: &Arc, source_file_path: Option<&Path>, @@ -998,13 +1041,6 @@ async fn persist_sanitized_source_config( Ok(doc) } -async fn sanitize_sources_for_persist_preserving_templates(mut source_config: SourcesConfigDto) -> SourcesConfigDto { - let templates = source_config.templates.take(); - let mut sanitized = sanitize_sources_for_persist(source_config).await; - sanitized.templates = templates; - sanitized -} - pub async fn sanitize_sources_for_persist(mut source_config: SourcesConfigDto) -> SourcesConfigDto { source_config.templates = None; for input in &mut source_config.inputs { @@ -1042,19 +1078,6 @@ pub async fn sanitize_sources_for_persist(mut source_config: SourcesConfigDto) - source_config } -pub async fn validate_and_persist_source_config( - app_config: &Arc, - dto: SourcesConfigDto, -) -> Result { - let templates_to_persist = validate_source_config_for_persist(app_config, &dto).await?; - - if let Some(template_definition) = templates_to_persist.as_ref() { - persist_templates_config(app_config, template_definition).await?; - } - - persist_source_config(app_config, None, dto).await -} - pub async fn persist_messaging_templates( app_config: &Arc, cfg: &mut ConfigDto, @@ -1271,11 +1294,11 @@ pub async fn migrate_api_user(api_proxy: &mut ApiProxyConfig, cfg: &AppConfig, e #[cfg(test)] mod tests { use super::{ - get_batch_aliases, prepare_sources_batch, sanitize_sources_for_persist, - sanitize_sources_for_persist_preserving_templates, write_config_file, + get_batch_aliases, prepare_sources_batch, sanitize_sources_for_persist, write_config_file, + write_config_text_file, }; use shared::{ - model::{ConfigInputAliasDto, ConfigInputDto, InputType, PatternTemplate, SourcesConfigDto, TemplateValue}, + model::{ConfigInputAliasDto, ConfigInputDto, InputType, SourcesConfigDto}, utils::Internable, }; use tempfile::tempdir; @@ -1348,28 +1371,6 @@ mod tests { assert!(std::fs::read_to_string(path).expect("read persisted csv").contains("updated")); } - #[tokio::test] - async fn internal_source_persist_keeps_expiry_update_and_inline_templates() -> Result<(), Box> - { - let templates = vec![PatternTemplate { - name: "channels".to_string(), - value: TemplateValue::Single("Input = provider".to_string()), - placeholder: String::new(), - }]; - let mut sources = SourcesConfigDto { - templates: Some(templates.clone()), - inputs: vec![ConfigInputDto { name: "provider".intern(), ..Default::default() }], - ..Default::default() - }; - assert!(sources.inputs[0].update_account_expiration_date("provider", 2_000_000_000, false)?); - - let sanitized = sanitize_sources_for_persist_preserving_templates(sources).await; - - assert_eq!(sanitized.templates, Some(templates)); - assert_eq!(sanitized.inputs[0].exp_date, Some(2_000_000_000)); - Ok(()) - } - #[tokio::test] async fn config_write_stops_when_backup_fails() -> Result<(), Box> { let dir = tempdir()?; @@ -1390,4 +1391,176 @@ mod tests { assert_eq!(tokio::fs::read_to_string(config_path).await?, "old"); Ok(()) } + + #[tokio::test] + async fn text_write_identical_content_is_noop() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + tokio::fs::write(&path, "same content").await?; + + let written = write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + "same content", + "source.yml", + None, + ) + .await?; + + assert!(!written); + assert_eq!(tokio::fs::read_to_string(&path).await?, "same content"); + Ok(()) + } + + #[tokio::test] + async fn text_write_replaces_content_and_creates_backup() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + let backup_dir = dir.path().join("backup"); + tokio::fs::write(&path, "old content").await?; + + let written = write_config_text_file( + path.to_string_lossy().as_ref(), + backup_dir.to_string_lossy().as_ref(), + "new content", + "source.yml", + None, + ) + .await?; + + assert!(written); + assert_eq!(tokio::fs::read_to_string(&path).await?, "new content"); + let mut backups: Vec<_> = std::fs::read_dir(&backup_dir)? + .filter_map(std::result::Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().starts_with("source.yml_")) + .map(|entry| entry.path()) + .collect(); + backups.sort(); + assert_eq!(backups.len(), 1); + assert_eq!(tokio::fs::read_to_string(&backups[0]).await?, "old content"); + Ok(()) + } + + #[tokio::test] + async fn text_write_preserves_crlf_and_unicode() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + let content = "key: \"héllo wörld\"\r\nother: value\r\n"; + tokio::fs::write(&path, "placeholder").await?; + + write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + content, + "source.yml", + None, + ) + .await?; + + assert_eq!(tokio::fs::read_to_string(&path).await?, content); + Ok(()) + } + + #[tokio::test] + async fn text_write_revision_conflict_leaves_file_untouched() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + tokio::fs::write(&path, "original").await?; + let stale_revision = blake3::hash(b"different content"); + + let result = write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + "patched", + "source.yml", + Some(stale_revision), + ) + .await; + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("changed while an internal patch was being prepared")); + assert_eq!(tokio::fs::read_to_string(&path).await?, "original"); + Ok(()) + } + + #[tokio::test] + async fn text_write_matching_revision_succeeds() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + tokio::fs::write(&path, "original").await?; + let revision = blake3::hash(b"original"); + + let written = write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + "patched", + "source.yml", + Some(revision), + ) + .await?; + + assert!(written); + assert_eq!(tokio::fs::read_to_string(&path).await?, "patched"); + Ok(()) + } + + #[tokio::test] + async fn text_write_matching_revision_and_identical_content_is_noop() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + tokio::fs::write(&path, "original").await?; + let revision = blake3::hash(b"original"); + + let written = write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + "original", + "source.yml", + Some(revision), + ) + .await?; + + assert!(!written); + assert!(!dir.path().join("backup").exists()); + Ok(()) + } + + #[tokio::test] + async fn text_write_missing_destination_is_a_revision_conflict() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + let revision = blake3::hash(b"original"); + + let result = write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + "patched", + "source.yml", + Some(revision), + ) + .await; + + assert!(result.is_err()); + assert!(!path.exists()); + Ok(()) + } + + #[tokio::test] + async fn text_write_preserves_missing_final_newline() -> Result<(), Box> { + let dir = tempdir()?; + let path = dir.path().join("source.yml"); + tokio::fs::write(&path, "old\n").await?; + + write_config_text_file( + path.to_string_lossy().as_ref(), + dir.path().join("backup").to_string_lossy().as_ref(), + "key: value", + "source.yml", + None, + ) + .await?; + + assert_eq!(tokio::fs::read_to_string(&path).await?, "key: value"); + Ok(()) + } } diff --git a/backend/config-loader/src/source_patch.rs b/backend/config-loader/src/source_patch.rs new file mode 100644 index 000000000..3ff713a33 --- /dev/null +++ b/backend/config-loader/src/source_patch.rs @@ -0,0 +1,1234 @@ +use serde::Deserialize; +use shared::{ + error::TuliproxError, + model::{ConfigInputAliasDto, SourcesConfigDto}, + utils::is_nullish, +}; +use std::{collections::HashMap, fmt::Write, ops::Range}; + +#[derive(Debug, Clone)] +pub struct TextEdit { + pub range: Range, + pub replacement: String, +} + +#[derive(Debug, Deserialize)] +pub struct SourcePatchDocument { + #[serde(default)] + pub inputs: Vec>, +} + +#[derive(Debug, Deserialize)] +pub struct PatchInput { + pub name: serde_saphyr::Spanned, + #[serde(default)] + pub enabled: Option>, + #[serde(default)] + pub url: Option>, + #[serde(default)] + pub username: Option>, + #[serde(default)] + pub password: Option>, + #[serde(default)] + pub exp_date: Option>, + #[serde(default)] + pub max_connections: Option>, + #[serde(default)] + pub panel_api: Option>, + #[serde(default)] + pub aliases: Option>>>, +} + +#[derive(Debug, Deserialize)] +pub struct PatchAlias { + pub name: serde_saphyr::Spanned, + #[serde(default)] + pub enabled: Option>, + #[serde(default)] + pub url: Option>, + #[serde(default)] + pub username: Option>, + #[serde(default)] + pub password: Option>, + #[serde(default)] + pub exp_date: Option>, + #[serde(default)] + pub max_connections: Option>, +} + +#[derive(Debug, Deserialize)] +pub struct PatchPanelApi { + #[serde(default)] + pub credits: Option>, +} + +pub fn span_byte_range(spanned: &serde_saphyr::Spanned) -> Result, TuliproxError> { + let loc = &spanned.referenced; + if loc != &spanned.defined { + return Err(TuliproxError::Config( + "source.yml patch rejected: value originates from a YAML alias or merge key".to_string(), + )); + } + let span = loc.span(); + let byte_offset = span.byte_offset().ok_or_else(|| { + TuliproxError::Config("source.yml patch rejected: byte span information unavailable".to_string()) + })?; + let byte_len = span.byte_len().ok_or_else(|| { + TuliproxError::Config("source.yml patch rejected: byte span information unavailable".to_string()) + })?; + let start = usize::try_from(byte_offset) + .map_err(|_| TuliproxError::Config("source.yml patch rejected: byte offset overflow".to_string()))?; + let len = usize::try_from(byte_len) + .map_err(|_| TuliproxError::Config("source.yml patch rejected: byte length overflow".to_string()))?; + let end = start + .checked_add(len) + .ok_or_else(|| TuliproxError::Config("source.yml patch rejected: byte range overflow".to_string()))?; + Ok(start..end) +} + +pub fn serialize_yaml_scalar(value: &T) -> Result { + let mut out = String::new(); + serde_saphyr::to_fmt_writer(&mut out, value) + .map_err(|err| TuliproxError::Config(format!("source.yml patch: scalar serialization failed: {err}")))?; + if out.ends_with('\n') { + out.pop(); + } + if out.ends_with('\r') { + out.pop(); + } + Ok(out) +} + +pub fn detect_newline(text: &str) -> &'static str { + if text.contains("\r\n") { + "\r\n" + } else { + "\n" + } +} + +pub fn line_indent_at(text: &str, byte_pos: usize) -> usize { + let line_start = text[..byte_pos].rfind('\n').map_or(0, |p| p + 1); + text[line_start..].chars().take_while(|c| *c == ' ' || *c == '\t').count() +} + +pub fn line_end_offset(text: &str, byte_pos: usize) -> usize { + match text[byte_pos..].find('\n') { + Some(off) => byte_pos + off + 1, + None => text.len(), + } +} + +/// Returns true when the mapping owning `byte_range` is written in flow style (`{...}`). +/// +/// Scanning starts at the beginning of the line holding the value and walks backwards over +/// preceding lines while they belong to the same flow scope, so a value on a continuation line +/// of a multi-line flow mapping is still detected. +fn is_flow_style_mapping_at(text: &str, byte_range: &Range) -> bool { + let mut depth = 0i32; + for ch in text[..byte_range.start.min(text.len())].chars().rev() { + match ch { + '}' | ']' => depth += 1, + '{' | '[' if depth == 0 => return true, + '{' | '[' => depth -= 1, + '\n' if depth == 0 => break, + _ => {} + } + } + false +} + +/// Rejects inserting a new key into a flow-style mapping. +/// +/// Replacing an existing scalar inside a flow mapping is safe because its span is explicit, +/// but inserting a whole `key: value` line is not representable there. +pub fn ensure_block_style_for_insertion( + text: &str, + anchor: &Range, + field_name: &str, +) -> Result<(), TuliproxError> { + if is_flow_style_mapping_at(text, anchor) { + return Err(TuliproxError::Config(format!( + "cannot insert optional field '{field_name}' into a flow-style YAML mapping; edit the account in block style" + ))); + } + Ok(()) +} + +/// Byte offset of the end of the line holding `byte_pos`, excluding its line break. +pub fn line_content_end(text: &str, byte_pos: usize) -> usize { + let line_end = line_end_offset(text, byte_pos); + let mut end = line_end; + if text[..end].ends_with('\n') { + end -= 1; + if text[..end].ends_with('\r') { + end -= 1; + } + } + end +} + +/// Column at which sibling keys of the mapping owning `byte_pos` start. +/// +/// For a sequence item written as ` - name: account` the siblings of `name` are indented to +/// the column of `name`, not to the column of the `-`. +pub fn sibling_key_indent(text: &str, byte_pos: usize) -> usize { + let line_start = text[..byte_pos.min(text.len())].rfind('\n').map_or(0, |p| p + 1); + let line = &text[line_start..]; + let mut indent = 0; + let mut chars = line.chars(); + for ch in chars.by_ref() { + match ch { + ' ' | '\t' => indent += 1, + '-' => { + indent += 1; + // Consume the whitespace that separates the dash from the first key. + for next in chars.by_ref() { + if next == ' ' || next == '\t' { + indent += 1; + } else { + break; + } + } + return indent; + } + _ => return indent, + } + } + indent +} + +/// Builds an edit that inserts `key: value` on its own line directly below the anchor line. +/// +/// The anchor range must point at an existing sibling scalar in the same mapping; its line +/// indentation and the document's newline style are reused so no existing byte changes. +pub fn build_field_insertion_edit( + text: &str, + anchor: &Range, + key: &str, + value: &str, +) -> Result { + ensure_block_style_for_insertion(text, anchor, key)?; + let insert_at = line_content_end(text, anchor.end); + let indent = sibling_key_indent(text, anchor.start); + let newline = detect_newline(text); + let replacement = format!("{newline}{:indent$}{key}: {value}", "", indent = indent); + Ok(TextEdit { range: insert_at..insert_at, replacement }) +} + +pub fn apply_scalar_edits(original_text: &str, edits: Vec) -> Result { + let mut sorted_edits: Vec = edits; + sorted_edits.sort_by_key(|edit| std::cmp::Reverse(edit.range.start)); + + for window in sorted_edits.windows(2) { + if window[0].range.start < window[1].range.end { + return Err(TuliproxError::Config("source.yml patch rejected: overlapping edit ranges".to_string())); + } + } + + let mut result = original_text.to_string(); + for edit in &sorted_edits { + if edit.range.end > result.len() { + return Err(TuliproxError::Config(format!( + "source.yml patch rejected: edit range {}..{} exceeds document length {}", + edit.range.start, + edit.range.end, + result.len() + ))); + } + result.replace_range(edit.range.clone(), &edit.replacement); + } + + Ok(result) +} + +pub fn parse_and_validate_patched_text(patched_text: &str, expected: &SourcesConfigDto) -> Result<(), TuliproxError> { + let mut reparsed: SourcesConfigDto = serde_saphyr::from_str(patched_text) + .map_err(|err| TuliproxError::Config(format!("patched source.yml failed to parse: {err}")))?; + let mut expected = expected.clone(); + + normalize_computed_source_fields(&mut reparsed); + normalize_computed_source_fields(&mut expected); + + if reparsed != expected { + return Err(TuliproxError::Config( + "patched source.yml did not match the expected configuration; original file was not changed".to_string(), + )); + } + Ok(()) +} + +/// Removes fields computed while preparing a source configuration and therefore not part of +/// persistent YAML semantics. Empty and missing alias collections are equivalent because an +/// empty `aliases:` mapping reparses as `None`. +fn normalize_computed_source_fields(dto: &mut SourcesConfigDto) { + for input in &mut dto.inputs { + input.id = 0; + if input.aliases.as_ref().is_some_and(Vec::is_empty) { + input.aliases = None; + } + if let Some(aliases) = input.aliases.as_mut() { + for alias in aliases { + alias.id = 0; + } + } + } +} + +pub fn parse_patch_document(text: &str) -> Result { + serde_saphyr::from_str(text) + .map_err(|err| TuliproxError::Config(format!("source.yml patch: projection parse failed: {err}"))) +} + +pub fn find_input<'a>( + doc: &'a SourcePatchDocument, + input_name: &str, +) -> Result<&'a serde_saphyr::Spanned, TuliproxError> { + let mut found: Option<&serde_saphyr::Spanned> = None; + for input in &doc.inputs { + if input.value.name.value == input_name { + if found.is_some() { + return Err(TuliproxError::Config(format!( + "source.yml patch target input '{input_name}' is ambiguous" + ))); + } + found = Some(input); + } + } + let input = found + .ok_or_else(|| TuliproxError::Config(format!("source.yml patch target input '{input_name}' was not found")))?; + ensure_unique_alias_names(input, input_name)?; + Ok(input) +} + +/// Rejects an input whose alias list contains the same name twice, because every alias lookup +/// during patch planning would then silently target the first match. +fn ensure_unique_alias_names(input: &serde_saphyr::Spanned, input_name: &str) -> Result<(), TuliproxError> { + let Some(aliases) = &input.value.aliases else { + return Ok(()); + }; + let mut seen: Vec<&str> = Vec::with_capacity(aliases.value.len()); + for alias in &aliases.value { + let name = alias.value.name.value.as_str(); + if seen.contains(&name) { + return Err(TuliproxError::Config(format!( + "source.yml patch target alias '{name}' under input '{input_name}' is ambiguous" + ))); + } + seen.push(name); + } + Ok(()) +} + +pub fn alias_item_block_range( + text: &str, + alias_span: &serde_saphyr::Spanned, +) -> Result, TuliproxError> { + let alias_range = span_byte_range(alias_span)?; + let name_range = span_byte_range(&alias_span.value.name)?; + let line_start = text[..alias_range.start].rfind('\n').map_or(0, |p| p + 1); + let dash_pos = text[line_start..alias_range.start].find('-').map_or(line_start, |p| line_start + p); + let block_start = text[..dash_pos].rfind('\n').map_or(0, |p| p + 1); + + let comment_start = find_owned_comment_start(text, block_start, line_start); + + let block_end; + let item_indent = text[block_start..].chars().take_while(|c| *c == ' ').count(); + + let mut pos = line_end_offset(text, name_range.end); + loop { + if pos >= text.len() { + block_end = text.len(); + break; + } + let next_line_start = pos; + let next_line_end = match text[next_line_start..].find('\n') { + Some(off) => next_line_start + off + 1, + None => text.len(), + }; + let next_line = &text[next_line_start..next_line_end.min(text.len())]; + let stripped = next_line.trim_start(); + if stripped.is_empty() { + pos = next_line_end; + continue; + } + let next_indent = next_line.chars().take_while(|c| *c == ' ').count(); + if next_indent < item_indent || (next_indent == item_indent && stripped.starts_with('-')) { + block_end = next_line_start; + break; + } + if next_indent == item_indent && stripped.starts_with('#') { + let mut lookahead = next_line_end; + let mut comments_belong_to_next_item = false; + while lookahead < text.len() { + let lookahead_end = match text[lookahead..].find('\n') { + Some(off) => lookahead + off + 1, + None => text.len(), + }; + let candidate = &text[lookahead..lookahead_end.min(text.len())]; + let candidate_stripped = candidate.trim_start(); + let candidate_indent = candidate.chars().take_while(|c| *c == ' ').count(); + if candidate_indent == item_indent && candidate_stripped.starts_with('#') { + lookahead = lookahead_end; + continue; + } + if candidate_indent == item_indent && candidate_stripped.starts_with('-') { + comments_belong_to_next_item = true; + break; + } + break; + } + if comments_belong_to_next_item { + block_end = next_line_start; + break; + } + } + pos = next_line_end; + } + + Ok(comment_start..block_end) +} + +fn find_owned_comment_start(text: &str, block_start: usize, _first_content_line_start: usize) -> usize { + let mut comment_start = block_start; + let mut pos = block_start; + while pos > 0 { + let prev_line_end = pos; + let prev_line_start = text[..prev_line_end.saturating_sub(1)].rfind('\n').map_or(0, |p| p + 1); + let prev_line = &text[prev_line_start..prev_line_end.min(text.len())]; + let stripped = prev_line.trim(); + if stripped.starts_with('#') { + let prev_indent = prev_line.chars().take_while(|c| *c == ' ').count(); + let block_indent = text[block_start..].chars().take_while(|c| *c == ' ').count(); + if prev_indent >= block_indent { + comment_start = prev_line_start; + pos = prev_line_start; + } else { + break; + } + } else { + break; + } + } + comment_start +} + +pub fn serialize_alias_block( + alias: &ConfigInputAliasDto, + indent: usize, + newline: &str, +) -> Result { + let pad = " ".repeat(indent); + let mut out = String::new(); + let _ = writeln!(out, "{pad}- name: {}", serialize_yaml_scalar(&alias.name.as_ref())?); + let _ = writeln!(out, "{pad} url: {}", serialize_yaml_scalar(&alias.url)?); + if let Some(username) = &alias.username { + let _ = writeln!(out, "{pad} username: {}", serialize_yaml_scalar(username)?); + } + if let Some(password) = &alias.password { + let _ = writeln!(out, "{pad} password: {}", serialize_yaml_scalar(password)?); + } + if alias.max_connections != 0 { + let _ = writeln!(out, "{pad} max_connections: {}", serialize_yaml_scalar(&alias.max_connections)?); + } + if let Some(exp_date) = alias.exp_date { + let _ = writeln!(out, "{pad} exp_date: {}", serialize_yaml_scalar(&exp_date)?); + } + if !alias.enabled { + let _ = writeln!(out, "{pad} enabled: false"); + } + if out.ends_with('\n') { + out.truncate(out.len() - 1); + } + if newline == "\r\n" { + out = out.replace('\n', "\r\n"); + } + Ok(out) +} + +pub fn build_alias_addition_edit( + text: &str, + doc: &SourcePatchDocument, + input_name: &str, + alias: &ConfigInputAliasDto, +) -> Result { + let input = find_input(doc, input_name)?; + let newline = detect_newline(text); + + if let Some(aliases_spanned) = &input.value.aliases { + let aliases = &aliases_spanned.value; + if aliases.is_empty() { + let aliases_range = span_byte_range(aliases_spanned)?; + let line_end = line_end_offset(text, aliases_range.end); + let indent = line_indent_at(text, aliases_range.start) + 2; + let block = serialize_alias_block(alias, indent, newline)?; + let insertion = format!("{newline}{block}{newline}"); + return Ok(TextEdit { range: line_end..line_end, replacement: insertion }); + } + let Some(last_alias) = aliases.last() else { + return Err(TuliproxError::Config(format!( + "source.yml patch: alias list for input '{input_name}' unexpectedly empty" + ))); + }; + let last_block = alias_item_block_range(text, last_alias)?; + let item_indent = text[last_block.start..].chars().take_while(|c| *c == ' ').count(); + let block = serialize_alias_block(alias, item_indent, newline)?; + let insertion = format!("{block}{newline}"); + return Ok(TextEdit { range: last_block.end..last_block.end, replacement: insertion }); + } + + let input_span_range = span_byte_range(&input.value.name)?; + let input_indent = line_indent_at(text, input_span_range.start); + let aliases_indent = input_indent; + let item_indent = aliases_indent + 2; + + // `aliases: null` (or an empty `aliases:`) deserializes to `None` but still occupies a line. + // Replacing that line in place is what turns the null marker into a real block sequence. + let field_indent = sibling_key_indent(text, input_span_range.start); + if let Some(null_line) = find_null_aliases_line(text, input_span_range.end, field_indent) { + let block = serialize_alias_block(alias, field_indent + 2, newline)?; + let pad = " ".repeat(field_indent); + let replacement = format!("{pad}aliases:{newline}{block}"); + return Ok(TextEdit { range: null_line, replacement }); + } + + let mut last_field_end: Option = None; + if let Some(f) = &input.value.panel_api { + last_field_end = Some(span_byte_range(f)?.end); + } + if last_field_end.is_none() { + if let Some(f) = &input.value.max_connections { + last_field_end = Some(span_byte_range(f)?.end); + } + } + if last_field_end.is_none() { + if let Some(f) = &input.value.exp_date { + last_field_end = Some(span_byte_range(f)?.end); + } + } + if last_field_end.is_none() { + if let Some(f) = &input.value.password { + last_field_end = Some(span_byte_range(f)?.end); + } + } + if last_field_end.is_none() { + if let Some(f) = &input.value.username { + last_field_end = Some(span_byte_range(f)?.end); + } + } + if last_field_end.is_none() { + if let Some(f) = &input.value.url { + last_field_end = Some(span_byte_range(f)?.end); + } + } + + let anchor_end = last_field_end.unwrap_or(input_span_range.end); + let line_end = line_end_offset(text, anchor_end); + let block = serialize_alias_block(alias, item_indent, newline)?; + let insertion = + format!("{newline}{:aliases_indent$}aliases:{newline}{block}{newline}", "", aliases_indent = aliases_indent); + Ok(TextEdit { range: line_end..line_end, replacement: insertion }) +} + +/// Scans forward from `start` for an `aliases:` line at `key_indent` whose value is empty or +/// the literal `null`. Returns the byte range of the line content (excluding the trailing newline). +fn find_null_aliases_line(text: &str, start: usize, key_indent: usize) -> Option> { + let mut line_start = start; + while line_start < text.len() { + let next_nl = text[line_start..].find('\n').map_or(text.len(), |p| line_start + p); + let line = &text[line_start..next_nl]; + if line.trim().is_empty() { + line_start = next_nl + 1; + continue; + } + let indent = line.chars().take_while(|c| *c == ' ').count(); + if indent < key_indent { + return None; + } + if indent != key_indent { + line_start = next_nl + 1; + continue; + } + let body = line[indent..].trim_start(); + if let Some(value) = body.strip_prefix("aliases:") { + let value = value.trim(); + if is_nullish(value) { + return Some(line_start..next_nl); + } + return None; + } + line_start = next_nl + 1; + } + None +} + +pub fn build_alias_removal_edits( + text: &str, + doc: &SourcePatchDocument, + input_name: &str, + alias_names_to_remove: &[&str], +) -> Result, TuliproxError> { + let input = find_input(doc, input_name)?; + let Some(aliases_spanned) = &input.value.aliases else { + return Ok(Vec::new()); + }; + + let mut edits = Vec::new(); + for alias_span in &aliases_spanned.value { + if alias_names_to_remove.contains(&alias_span.value.name.value.as_str()) { + let block = alias_item_block_range(text, alias_span)?; + edits.push(TextEdit { range: block, replacement: String::new() }); + } + } + Ok(edits) +} + +pub fn build_alias_sort_edit( + text: &str, + doc: &SourcePatchDocument, + input_name: &str, + sorted_alias_names: &[&str], +) -> Result, TuliproxError> { + let input = find_input(doc, input_name)?; + let Some(aliases_spanned) = &input.value.aliases else { + return Ok(None); + }; + let aliases = &aliases_spanned.value; + if aliases.len() < 2 { + return Ok(None); + } + + let current_order: Vec<&str> = aliases.iter().map(|a| a.value.name.value.as_str()).collect(); + if current_order == sorted_alias_names { + return Ok(None); + } + + let mut blocks: Vec<(String, Range)> = Vec::new(); + for alias_span in aliases { + let block_range = alias_item_block_range(text, alias_span)?; + blocks.push((alias_span.value.name.value.clone(), block_range)); + } + + let (Some(first_block_start), Some(last_block_end)) = + (blocks.iter().map(|(_, r)| r.start).min(), blocks.iter().map(|(_, r)| r.end).max()) + else { + return Ok(None); + }; + + let mut reordered_text = String::new(); + for name in sorted_alias_names { + let (_, range) = blocks.iter().find(|(n, _)| n == name).ok_or_else(|| { + TuliproxError::Config(format!( + "source.yml patch: alias '{name}' not found during sort for input '{input_name}'" + )) + })?; + reordered_text.push_str(&text[range.clone()]); + } + + Ok(Some(TextEdit { range: first_block_start..last_block_end, replacement: reordered_text })) +} + +/// Rebuilds an alias sequence from opaque existing blocks and newly serialized aliases. +/// +/// Existing alias contents and their owned comments are copied byte-for-byte. This is used when +/// one semantic command both changes the alias set and determines a new order, which cannot be +/// represented safely as independent edits against the same original spans. +pub fn build_alias_sequence_edit( + text: &str, + doc: &SourcePatchDocument, + input_name: &str, + expected_aliases: &[ConfigInputAliasDto], +) -> Result, TuliproxError> { + let input = find_input(doc, input_name)?; + let Some(aliases_spanned) = &input.value.aliases else { + return Ok(None); + }; + let aliases = &aliases_spanned.value; + if aliases.is_empty() { + return Ok(None); + } + + let mut blocks = HashMap::<&str, Range>::with_capacity(aliases.len()); + for alias in aliases { + blocks.insert(alias.value.name.value.as_str(), alias_item_block_range(text, alias)?); + } + let start = blocks + .values() + .map(|range| range.start) + .min() + .ok_or_else(|| TuliproxError::Config("source.yml patch: alias sequence has no start".to_string()))?; + let end = blocks + .values() + .map(|range| range.end) + .max() + .ok_or_else(|| TuliproxError::Config("source.yml patch: alias sequence has no end".to_string()))?; + let first_name_start = + aliases.first().map(|alias| span_byte_range(&alias.value.name)).transpose()?.map_or(start, |range| range.start); + let item_indent = line_indent_at(text, first_name_start); + let newline = detect_newline(text); + let mut replacement = String::new(); + + for alias in expected_aliases { + if let Some(range) = blocks.get(alias.name.as_ref()) { + replacement.push_str(&text[range.clone()]); + } else { + replacement.push_str(&serialize_alias_block(alias, item_indent, newline)?); + replacement.push_str(newline); + } + } + + Ok(Some(TextEdit { range: start..end, replacement })) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FIXTURE: &str = concat!( + "templates:\n", + " - name: provider_channels\n", + " value: 'Input = \"provider\"'\n", + "\n", + "inputs:\n", + " - name: provider\n", + " enabled: true # must survive\n", + " type: xtream\n", + " url: provider://main\n", + " username: ${env:XTREAM_USER}\n", + " password: ${env:XTREAM_PASSWORD}\n", + " aliases:\n", + " # backup account comment follows the account when sorted\n", + " - name: provider-backup\n", + " url: http://backup.example\n", + " username: backup\n", + " password: \"contains: # special characters\"\n", + "\n", + "sources:\n", + " - inputs:\n", + " - provider\n", + " targets:\n", + " - name: output\n", + " filter: \"!provider_channels!\"\n", + " output:\n", + " - type: m3u\n", + ); + + #[test] + fn projection_parses_fixture() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + assert_eq!(doc.inputs.len(), 1); + assert_eq!(doc.inputs[0].value.name.value, "provider"); + assert!(doc.inputs[0].value.enabled.is_some()); + assert!(doc.inputs[0].value.exp_date.is_none()); + let aliases = doc.inputs[0].value.aliases.as_ref().expect("aliases"); + assert_eq!(aliases.value.len(), 1); + assert_eq!(aliases.value[0].value.name.value, "provider-backup"); + } + + #[test] + fn span_byte_ranges_are_valid() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let name_range = span_byte_range(&doc.inputs[0].value.name).expect("byte range"); + assert_eq!(&FIXTURE[name_range.clone()], "provider"); + + let enabled_range = + span_byte_range(doc.inputs[0].value.enabled.as_ref().expect("enabled")).expect("byte range"); + assert_eq!(&FIXTURE[enabled_range], "true"); + } + + #[test] + fn existing_scalar_replacement_preserves_surrounding_bytes() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let enabled_range = + span_byte_range(doc.inputs[0].value.enabled.as_ref().expect("enabled")).expect("byte range"); + + let new_value = serialize_yaml_scalar(&false).expect("serialize"); + let edit = TextEdit { range: enabled_range, replacement: new_value }; + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + let expected = FIXTURE.replace("enabled: true # must survive", "enabled: false # must survive"); + assert_eq!(patched, expected); + } + + #[test] + fn missing_exp_date_insertion_preserves_all_existing_bytes() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let input = &doc.inputs[0].value; + + let password_range = span_byte_range(input.password.as_ref().expect("password")).expect("byte range"); + let anchor_end = line_end_offset(FIXTURE, password_range.end); + let indent = line_indent_at(FIXTURE, password_range.start); + let newline = detect_newline(FIXTURE); + let new_value = serialize_yaml_scalar(&1_900_000_000_i64).expect("serialize"); + let insertion = format!("{newline}{:indent$}exp_date: {new_value}", "", indent = indent); + + let edit = TextEdit { range: anchor_end..anchor_end, replacement: insertion }; + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + assert!(patched.contains("exp_date: 1900000000")); + for line in FIXTURE.lines() { + assert!(patched.contains(line), "line missing after patch: {line}"); + } + } + + #[test] + fn alias_exp_date_insertion() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let aliases = doc.inputs[0].value.aliases.as_ref().expect("aliases"); + let alias = &aliases.value[0].value; + + let password_range = span_byte_range(alias.password.as_ref().expect("password")).expect("byte range"); + let anchor_end = line_end_offset(FIXTURE, password_range.end); + let indent = line_indent_at(FIXTURE, password_range.start); + let newline = detect_newline(FIXTURE); + let new_value = serialize_yaml_scalar(&2_000_000_000_i64).expect("serialize"); + let insertion = format!("{newline}{:indent$}exp_date: {new_value}", "", indent = indent); + + let edit = TextEdit { range: anchor_end..anchor_end, replacement: insertion }; + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + assert!(patched.contains("exp_date: 2000000000")); + assert!(patched.contains("password: \"contains: # special characters\"")); + } + + #[test] + fn inline_comment_after_scalar_survives_replacement() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let enabled_range = + span_byte_range(doc.inputs[0].value.enabled.as_ref().expect("enabled")).expect("byte range"); + + let new_value = serialize_yaml_scalar(&false).expect("serialize"); + let edit = TextEdit { range: enabled_range, replacement: new_value }; + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + assert!(patched.contains("enabled: false # must survive")); + } + + #[test] + fn crlf_preservation() { + let crlf_fixture = FIXTURE.replace('\n', "\r\n"); + let doc: SourcePatchDocument = serde_saphyr::from_str(&crlf_fixture).expect("projection parses"); + let enabled_range = + span_byte_range(doc.inputs[0].value.enabled.as_ref().expect("enabled")).expect("byte range"); + + let new_value = serialize_yaml_scalar(&false).expect("serialize"); + let edit = TextEdit { range: enabled_range, replacement: new_value }; + let patched = apply_scalar_edits(&crlf_fixture, vec![edit]).expect("apply"); + + assert!(patched.contains("\r\n")); + assert!(patched.contains("enabled: false # must survive")); + } + + #[test] + fn unicode_before_span_does_not_corrupt_offsets() { + let unicode_fixture = "# Ünïcödé tëst 🎉\ninputs:\n - name: über-input\n exp_date: 12345\n"; + let doc: SourcePatchDocument = serde_saphyr::from_str(unicode_fixture).expect("projection parses"); + let exp_range = span_byte_range(doc.inputs[0].value.exp_date.as_ref().expect("exp_date")).expect("byte range"); + assert_eq!(&unicode_fixture[exp_range.clone()], "12345"); + + let new_value = serialize_yaml_scalar(&99999_i64).expect("serialize"); + let edit = TextEdit { range: exp_range, replacement: new_value }; + let patched = apply_scalar_edits(unicode_fixture, vec![edit]).expect("apply"); + assert!(patched.contains("exp_date: 99999")); + assert!(patched.starts_with("# Ünïcödé tëst 🎉\n")); + } + + #[test] + fn duplicate_input_names_fail_without_output() { + let dup_fixture = "inputs:\n - name: same\n exp_date: 1\n - name: same\n exp_date: 2\n"; + let doc: SourcePatchDocument = serde_saphyr::from_str(dup_fixture).expect("projection parses"); + let names: Vec<&str> = doc.inputs.iter().map(|i| i.value.name.value.as_str()).collect(); + let unique: std::collections::HashSet<&str> = names.iter().copied().collect(); + assert!(unique.len() < names.len(), "duplicate names detected"); + } + + #[test] + fn yaml_anchor_merge_target_fails() { + let anchor_fixture = "defaults: &defaults\n exp_date: 100\ninputs:\n - name: test\n <<: *defaults\n"; + let doc: SourcePatchDocument = serde_saphyr::from_str(anchor_fixture).expect("projection parses"); + let input = &doc.inputs[0].value; + if let Some(exp_date) = &input.exp_date { + let result = span_byte_range(exp_date); + assert!(result.is_err(), "merge-derived value should be rejected"); + } + } + + #[test] + fn overlapping_edits_are_rejected() { + let edits = vec![ + TextEdit { range: 5..15, replacement: "a".to_string() }, + TextEdit { range: 10..20, replacement: "b".to_string() }, + ]; + let result = apply_scalar_edits("01234567890123456789", edits); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("overlapping")); + } + + #[test] + fn flow_style_scalar_replacement_succeeds() { + let flow_fixture = "inputs:\n - {name: test, exp_date: 42}\n"; + let doc: SourcePatchDocument = serde_saphyr::from_str(flow_fixture).expect("projection parses"); + let exp_range = span_byte_range(doc.inputs[0].value.exp_date.as_ref().expect("exp_date")).expect("byte range"); + assert_eq!(&flow_fixture[exp_range.clone()], "42"); + + let new_value = serialize_yaml_scalar(&99_i64).expect("serialize"); + let edit = TextEdit { range: exp_range, replacement: new_value }; + let patched = apply_scalar_edits(flow_fixture, vec![edit]).expect("apply"); + assert_eq!(patched, "inputs:\n - {name: test, exp_date: 99}\n"); + } + + #[test] + fn quoted_values_survive_unchanged() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let aliases = doc.inputs[0].value.aliases.as_ref().expect("aliases"); + let alias = &aliases.value[0].value; + let password_range = span_byte_range(alias.password.as_ref().expect("password")).expect("byte range"); + assert_eq!(&FIXTURE[password_range.clone()], "\"contains: # special characters\""); + + let new_value = serialize_yaml_scalar(&"new: pass").expect("serialize"); + let edit = TextEdit { range: password_range, replacement: new_value }; + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + let username_range = span_byte_range(alias.username.as_ref().expect("username")).expect("byte range"); + assert_eq!(&patched[username_range], "backup"); + } + + #[test] + fn semantic_validation_rejects_mismatch() { + let mut expected: SourcesConfigDto = serde_saphyr::from_str(FIXTURE).expect("parse"); + expected.inputs[0].exp_date = Some(999); + + let result = parse_and_validate_patched_text(FIXTURE, &expected); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("did not match")); + } + + #[test] + fn semantic_validation_accepts_matching_text() { + let expected: SourcesConfigDto = serde_saphyr::from_str(FIXTURE).expect("parse"); + let result = parse_and_validate_patched_text(FIXTURE, &expected); + assert!(result.is_ok()); + } + + #[test] + fn templates_and_filter_placeholder_survive_patch() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let password_range = + span_byte_range(doc.inputs[0].value.password.as_ref().expect("password")).expect("byte range"); + let anchor_end = line_end_offset(FIXTURE, password_range.end); + let indent = line_indent_at(FIXTURE, password_range.start); + let newline = detect_newline(FIXTURE); + let new_value = serialize_yaml_scalar(&1_800_000_000_i64).expect("serialize"); + let insertion = format!("{newline}{:indent$}exp_date: {new_value}", "", indent = indent); + + let edit = TextEdit { range: anchor_end..anchor_end, replacement: insertion }; + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + assert!(patched.contains("templates:\n - name: provider_channels\n value: 'Input = \"provider\"'")); + assert!(patched.contains("filter: \"!provider_channels!\"")); + assert!(patched.contains("username: ${env:XTREAM_USER}")); + assert!(patched.contains("password: ${env:XTREAM_PASSWORD}")); + } + + #[test] + fn add_alias_to_existing_list() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let new_alias = ConfigInputAliasDto { + name: "provider-second".into(), + url: "http://second.example".to_string(), + username: Some("second-user".to_string()), + password: Some("second-pass".to_string()), + exp_date: Some(2_000_000_000), + ..Default::default() + }; + + let edit = build_alias_addition_edit(FIXTURE, &doc, "provider", &new_alias).expect("build edit"); + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + assert!(patched.contains("- name: provider-second")); + assert!(patched.contains("url: http://second.example")); + assert!(patched.contains("username: second-user")); + assert!(patched.contains("password: second-pass")); + assert!(patched.contains("exp_date: 2000000000")); + for line in FIXTURE.lines() { + assert!(patched.contains(line), "line missing after patch: {line}"); + } + } + + #[test] + fn add_first_alias_when_aliases_absent() { + let no_aliases_fixture = "inputs:\n - name: solo\n url: http://solo.example\n"; + let doc: SourcePatchDocument = serde_saphyr::from_str(no_aliases_fixture).expect("projection parses"); + let new_alias = ConfigInputAliasDto { + name: "solo-alias".into(), + url: "http://alias.example".to_string(), + username: Some("alias-user".to_string()), + password: Some("alias-pass".to_string()), + ..Default::default() + }; + + let edit = build_alias_addition_edit(no_aliases_fixture, &doc, "solo", &new_alias).expect("build edit"); + let patched = apply_scalar_edits(no_aliases_fixture, vec![edit]).expect("apply"); + + assert!(patched.contains("aliases:")); + assert!(patched.contains("- name: solo-alias")); + assert!(patched.contains("url: http://alias.example")); + } + + #[test] + fn remove_alias_from_list() { + let multi_alias_fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " - name: first\n", + " url: http://first.example\n", + " - name: second\n", + " url: http://second.example\n", + " - name: third\n", + " url: http://third.example\n", + ); + let doc: SourcePatchDocument = serde_saphyr::from_str(multi_alias_fixture).expect("projection parses"); + + let edits = build_alias_removal_edits(multi_alias_fixture, &doc, "provider", &["second"]).expect("build edits"); + let patched = apply_scalar_edits(multi_alias_fixture, edits).expect("apply"); + + assert!(!patched.contains("second")); + assert!(patched.contains("first")); + assert!(patched.contains("third")); + } + + #[test] + fn sort_aliases_reorders_blocks() { + let sort_fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " # comment for oldest\n", + " - name: oldest\n", + " url: http://oldest.example\n", + " exp_date: 100\n", + " # comment for newest\n", + " - name: newest\n", + " url: http://newest.example\n", + " exp_date: 200\n", + ); + let doc: SourcePatchDocument = serde_saphyr::from_str(sort_fixture).expect("projection parses"); + + let edit = build_alias_sort_edit(sort_fixture, &doc, "provider", &["newest", "oldest"]) + .expect("build edit") + .expect("edit present"); + let patched = apply_scalar_edits(sort_fixture, vec![edit]).expect("apply"); + + let newest_pos = patched.find("name: newest").expect("newest present"); + let oldest_pos = patched.find("name: oldest").expect("oldest present"); + assert!(newest_pos < oldest_pos, "newest should come before oldest after sort"); + assert!(patched.contains("# comment for newest")); + assert!(patched.contains("# comment for oldest")); + } + + #[test] + fn sort_aliases_preserves_fields_before_name() { + let sort_fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " - url: http://oldest.example\n", + " enabled: false\n", + " name: oldest\n", + " exp_date: 100\n", + " - max_connections: 2\n", + " url: http://newest.example\n", + " name: newest\n", + " exp_date: 200\n", + ); + let doc: SourcePatchDocument = serde_saphyr::from_str(sort_fixture).expect("projection parses"); + + let edit = build_alias_sort_edit(sort_fixture, &doc, "provider", &["newest", "oldest"]) + .expect("build edit") + .expect("edit present"); + let patched = apply_scalar_edits(sort_fixture, vec![edit]).expect("apply"); + let reparsed: SourcePatchDocument = serde_saphyr::from_str(&patched).expect("reparse"); + let aliases = reparsed.inputs[0].value.aliases.as_ref().expect("aliases"); + + assert_eq!(aliases.value[0].value.name.value, "newest"); + assert_eq!(aliases.value[0].value.max_connections.as_ref().expect("max connections").value, 2); + assert_eq!(aliases.value[1].value.name.value, "oldest"); + assert_eq!(aliases.value[1].value.enabled.as_ref().expect("enabled").value, false); + } + + #[test] + fn remove_alias_removes_fields_before_name() { + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " aliases:\n", + " - url: http://remove.example\n", + " enabled: false\n", + " name: remove\n", + " - url: http://keep.example\n", + " name: keep\n", + ); + let doc: SourcePatchDocument = serde_saphyr::from_str(fixture).expect("projection parses"); + + let edits = build_alias_removal_edits(fixture, &doc, "provider", &["remove"]).expect("build edits"); + let patched = apply_scalar_edits(fixture, edits).expect("apply"); + let reparsed: SourcePatchDocument = serde_saphyr::from_str(&patched).expect("reparse"); + let aliases = reparsed.inputs[0].value.aliases.as_ref().expect("aliases"); + + assert_eq!(aliases.value.len(), 1); + assert_eq!(aliases.value[0].value.name.value, "keep"); + assert!(!patched.contains("http://remove.example")); + assert!(!patched.contains("enabled: false")); + } + + #[test] + fn alias_credentials_with_special_chars_serialize_correctly() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let new_alias = ConfigInputAliasDto { + name: "provider-special".into(), + url: "http://special.example".to_string(), + username: Some("user:with:colons".to_string()), + password: Some("pass # with hash".to_string()), + ..Default::default() + }; + + let edit = build_alias_addition_edit(FIXTURE, &doc, "provider", &new_alias).expect("build edit"); + let patched = apply_scalar_edits(FIXTURE, vec![edit]).expect("apply"); + + let reparsed: SourcePatchDocument = serde_saphyr::from_str(&patched).expect("reparse"); + let input = find_input(&reparsed, "provider").expect("find input"); + let aliases = input.value.aliases.as_ref().expect("aliases"); + let special = aliases.value.iter().find(|a| a.value.name.value == "provider-special").expect("special alias"); + assert_eq!(special.value.username.as_ref().expect("username").value, "user:with:colons"); + assert_eq!(special.value.password.as_ref().expect("password").value, "pass # with hash"); + } + + #[test] + fn combined_add_and_sort_produces_valid_transaction() { + let doc: SourcePatchDocument = serde_saphyr::from_str(FIXTURE).expect("projection parses"); + let new_alias = ConfigInputAliasDto { + name: "provider-new".into(), + url: "http://new.example".to_string(), + exp_date: Some(3_000_000_000), + ..Default::default() + }; + + let add_edit = build_alias_addition_edit(FIXTURE, &doc, "provider", &new_alias).expect("build add"); + let patched_after_add = apply_scalar_edits(FIXTURE, vec![add_edit]).expect("apply add"); + + let doc2: SourcePatchDocument = serde_saphyr::from_str(&patched_after_add).expect("reparse"); + let sort_edit = + build_alias_sort_edit(&patched_after_add, &doc2, "provider", &["provider-new", "provider-backup"]) + .expect("build sort") + .expect("sort edit present"); + let final_text = apply_scalar_edits(&patched_after_add, vec![sort_edit]).expect("apply sort"); + + assert!(final_text.contains("- name: provider-new")); + assert!(final_text.contains("- name: provider-backup")); + } + + #[test] + fn add_alias_when_aliases_is_null_replaces_line_in_place() { + let fixture = + concat!("inputs:\n", " - name: provider\n", " url: http://main.example\n", " aliases: null\n",); + let doc: SourcePatchDocument = serde_saphyr::from_str(fixture).expect("projection parses"); + let new_alias = ConfigInputAliasDto { + name: "provider-backup".into(), + url: "http://backup.example".to_string(), + ..Default::default() + }; + + let edit = build_alias_addition_edit(fixture, &doc, "provider", &new_alias).expect("build edit"); + let patched = apply_scalar_edits(fixture, vec![edit]).expect("apply"); + + assert!(!patched.contains("aliases: null"), "null marker must be gone"); + assert!(patched.contains("- name: provider-backup")); + assert!(patched.contains("url: http://backup.example")); + // Every original line (other than `aliases: null`) survives byte-for-byte. + for line in fixture.lines().filter(|l| !l.trim().is_empty() && *l != " aliases: null") { + assert!(patched.contains(line), "line missing after patch: {line}"); + } + } + + #[test] + fn add_alias_when_aliases_key_has_no_value_replaces_line_in_place() { + let fixture = concat!("inputs:\n", " - name: provider\n", " url: http://main.example\n", " aliases:\n",); + let doc: SourcePatchDocument = serde_saphyr::from_str(fixture).expect("projection parses"); + let new_alias = ConfigInputAliasDto { + name: "provider-backup".into(), + url: "http://backup.example".to_string(), + ..Default::default() + }; + + let edit = build_alias_addition_edit(fixture, &doc, "provider", &new_alias).expect("build edit"); + let patched = apply_scalar_edits(fixture, vec![edit]).expect("apply"); + + assert!(patched.contains("aliases:")); + assert!(patched.contains("- name: provider-backup")); + // Reparse must succeed and yield exactly one alias — guards against duplicate `aliases:` keys. + let reparsed: SourcePatchDocument = serde_saphyr::from_str(&patched).expect("reparse"); + let aliases = reparsed.inputs[0].value.aliases.as_ref().expect("aliases"); + assert_eq!(aliases.value.len(), 1); + } + + #[test] + fn duplicate_alias_names_are_rejected() { + let fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " - name: twin\n", + " url: http://first.example\n", + " - name: twin\n", + " url: http://second.example\n", + ); + let doc: SourcePatchDocument = serde_saphyr::from_str(fixture).expect("projection parses"); + + let err = find_input(&doc, "provider").expect_err("duplicate alias must be rejected"); + let msg = format!("{err:?}"); + assert!(msg.contains("alias 'twin'"), "error must name the duplicated alias, got: {msg}"); + } + + #[test] + fn sort_aliases_keeps_comments_attached_to_their_original_blocks() { + let sort_fixture = concat!( + "inputs:\n", + " - name: provider\n", + " url: http://main.example\n", + " aliases:\n", + " # comment for oldest\n", + " - name: oldest\n", + " url: http://oldest.example\n", + " exp_date: 100\n", + " # comment for newest\n", + " - name: newest\n", + " url: http://newest.example\n", + " exp_date: 200\n", + ); + let doc: SourcePatchDocument = serde_saphyr::from_str(sort_fixture).expect("projection parses"); + + let edit = build_alias_sort_edit(sort_fixture, &doc, "provider", &["newest", "oldest"]) + .expect("build edit") + .expect("edit present"); + let patched = apply_scalar_edits(sort_fixture, vec![edit]).expect("apply"); + + let newest_pos = patched.find("name: newest").expect("newest present"); + let oldest_pos = patched.find("name: oldest").expect("oldest present"); + let newest_comment_pos = patched.find("# comment for newest").expect("newest comment present"); + let oldest_comment_pos = patched.find("# comment for oldest").expect("oldest comment present"); + assert_eq!(patched.matches("# comment for newest").count(), 1); + assert_eq!(patched.matches("# comment for oldest").count(), 1); + assert!(newest_pos < oldest_pos, "newest should come before oldest after sort"); + // Comment must travel with its block, not get left in front of the other name. + assert!(newest_comment_pos < oldest_pos, "newest comment must precede oldest name after sort, not trail it"); + assert!( + oldest_comment_pos > newest_pos && oldest_comment_pos > newest_comment_pos, + "oldest comment must come after newest block, not before it" + ); + } +} diff --git a/backend/core/src/model/config/app.rs b/backend/core/src/model/config/app.rs index 38cf1b591..a2a3f705f 100644 --- a/backend/core/src/model/config/app.rs +++ b/backend/core/src/model/config/app.rs @@ -622,7 +622,7 @@ mod tests { name: "target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: crate::model::StagedFilter::default(), output: Vec::new(), rename: None, mapping_ids: Some(vec![mapping_id.to_string()]), diff --git a/backend/core/src/model/config/input.rs b/backend/core/src/model/config/input.rs index a5d3d98d9..96ff8d4b0 100644 --- a/backend/core/src/model/config/input.rs +++ b/backend/core/src/model/config/input.rs @@ -1012,13 +1012,12 @@ impl fmt::Display for ConfigInput { } pub fn is_input_expired(exp_date: Option) -> bool { - match exp_date { - Some(ts) => { - let now = Utc::now().timestamp(); - ts <= now - } - None => false, - } + let now = Utc::now().timestamp(); + u64::try_from(now).map_or_else(|_| exp_date.is_some(), |now| is_input_expired_at(exp_date, now)) +} + +pub fn is_input_expired_at(exp_date: Option, now: u64) -> bool { + exp_date.is_some_and(|timestamp| u64::try_from(timestamp).map_or(true, |timestamp| timestamp <= now)) } /// Resolves a custom "provider://" URL using a pre-provided provider configuration. diff --git a/backend/core/src/model/config/target.rs b/backend/core/src/model/config/target.rs index c146e3d3c..e1e3a7b31 100644 --- a/backend/core/src/model/config/target.rs +++ b/backend/core/src/model/config/target.rs @@ -61,6 +61,16 @@ impl TargetExecutionPlan { } } +#[derive(Debug, Clone, Default)] +pub struct StagedFilter { + pub processing: Option, + pub persist: Option, +} + +impl From for StagedFilter { + fn from(processing: Filter) -> Self { Self { processing: Some(processing), ..Self::default() } } +} + #[derive(Debug)] pub struct CompiledTargetMappings { pub all: Vec>, @@ -316,7 +326,7 @@ pub struct ConfigTarget { pub name: String, pub options: Option, pub sort: Option, - pub filter: Filter, + pub filter: StagedFilter, pub output: Vec, pub rename: Option>, pub mapping_ids: Option>, @@ -329,7 +339,9 @@ pub struct ConfigTarget { } impl ConfigTarget { - pub fn filter(&self, provider: &ValueProvider) -> bool { self.filter.filter(provider) } + pub fn filter(&self, provider: &ValueProvider) -> bool { + self.filter.processing.as_ref().is_none_or(|filter| filter.filter(provider)) + } pub fn get_xtream_output(&self) -> Option<&XtreamTargetOutput> { self.output.iter().find_map(|o| match o { @@ -374,7 +386,7 @@ impl From<&ConfigTargetDto> for ConfigTarget { name: dto.name.clone(), options: dto.options.clone(), sort: dto.sort.as_ref().map(Into::into), - filter: dto.t_filter.clone().unwrap_or_default(), + filter: StagedFilter { processing: dto.filter.t_processing.clone(), persist: dto.filter.t_persist.clone() }, output: dto.output.iter().map(Into::into).collect(), rename: dto.rename.as_ref().map(|l| l.iter().map(Into::into).collect()), mapping_ids: dto.mapping.clone(), diff --git a/backend/core/src/model/playlist_filter.rs b/backend/core/src/model/playlist_filter.rs index 3cbcdcd7c..25498f114 100644 --- a/backend/core/src/model/playlist_filter.rs +++ b/backend/core/src/model/playlist_filter.rs @@ -9,19 +9,38 @@ use shared::{ model::{PlaylistGroup, PlaylistItem}, }; +#[derive(Debug, Default, PartialEq, Eq)] +pub struct FilterOutcome { + pub inspected: usize, + pub retained: usize, + pub removed: usize, +} + +impl FilterOutcome { + pub fn record(&mut self, retained: bool) -> bool { + self.inspected += 1; + if retained { + self.retained += 1; + } else { + self.removed += 1; + } + retained + } +} + pub fn is_valid(pli: &PlaylistItem, filter: &Filter, match_as_ascii: bool) -> bool { let provider = ValueProvider { pli, match_as_ascii }; filter.filter(&provider) } -pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) -> Option> { +pub fn apply_filter_to_playlist(playlist: &[PlaylistGroup], filter: &Filter) -> Vec { // NOTE: the source `playlist` is intentionally cloned (not drained) here because // the caller reuses the same slice for every target output and for the no-filter // fallback path, so the survivors cannot be moved out of it. Cap the initial // allocation so selective filters do not retain capacity for every source item. const INITIAL_FILTERED_GROUP_CAPACITY: usize = 256; let mut new_playlist = Vec::with_capacity(playlist.len()); - for pg in playlist.iter() { + for pg in playlist { let mut channels = Vec::with_capacity(pg.channels.len().min(INITIAL_FILTERED_GROUP_CAPACITY)); channels.extend(pg.channels.iter().filter(|&pli| is_valid(pli, filter, false)).cloned()); if !channels.is_empty() { @@ -33,9 +52,58 @@ pub fn apply_filter_to_playlist(playlist: &mut [PlaylistGroup], filter: &Filter) }); } } - if new_playlist.is_empty() { - None - } else { - Some(new_playlist) + new_playlist +} + +pub fn retain_filtered_playlist(playlist: &mut Vec, filter: &Filter) -> FilterOutcome { + let mut outcome = FilterOutcome::default(); + for group in playlist.iter_mut() { + group.channels.retain(|item| outcome.record(is_valid(item, filter, false))); + } + playlist.retain(|group| !group.channels.is_empty()); + outcome +} + +#[cfg(test)] +mod tests { + use super::{apply_filter_to_playlist, retain_filtered_playlist, FilterOutcome}; + use shared::{ + foundation::get_filter, + model::{PlaylistGroup, PlaylistItem, PlaylistItemHeader, XtreamCluster}, + utils::Internable, + }; + + fn playlist_with_name(name: &str) -> Vec { + vec![PlaylistGroup { + id: 1, + title: "Group".intern(), + channels: vec![PlaylistItem { + header: PlaylistItemHeader { name: name.intern(), group: "Group".intern(), ..Default::default() }, + }], + xtream_cluster: XtreamCluster::Live, + }] + } + + #[test] + fn output_filter_with_no_match_returns_an_empty_playlist() { + let playlist = playlist_with_name("kept-by-persist"); + let filter = get_filter(r#"Name = "not-present""#, None).expect("filter"); + + let filtered = apply_filter_to_playlist(&playlist, &filter); + + assert!(filtered.is_empty()); + } + + #[test] + fn retain_filter_removes_empty_groups_and_reports_outcome() { + let mut playlist = playlist_with_name("removed"); + playlist.extend(playlist_with_name("kept")); + let filter = get_filter(r#"Name = "kept""#, None).expect("filter"); + + let outcome = retain_filtered_playlist(&mut playlist, &filter); + + assert_eq!(outcome, FilterOutcome { inspected: 2, retained: 1, removed: 1 }); + assert_eq!(playlist.len(), 1); + assert_eq!(playlist[0].channels.len(), 1); } } diff --git a/backend/core/src/model/xtream.rs b/backend/core/src/model/xtream.rs index cb2a3d824..70974c8da 100644 --- a/backend/core/src/model/xtream.rs +++ b/backend/core/src/model/xtream.rs @@ -134,7 +134,6 @@ mod tests { }; use arc_swap::{ArcSwap, ArcSwapOption}; use shared::{ - foundation::Filter, model::{ConfigPaths, InputFetchMethod, InputType, PlaylistItemType, ProcessingOrder}, utils::Internable, }; @@ -190,7 +189,7 @@ mod tests { name: "xtream-target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: crate::model::StagedFilter::default(), output: vec![TargetOutput::Xtream(xtream_output.clone())], rename: None, mapping_ids: None, diff --git a/backend/dvr/src/download.rs b/backend/dvr/src/download.rs index 8448bb695..095ead0b0 100644 --- a/backend/dvr/src/download.rs +++ b/backend/dvr/src/download.rs @@ -1174,74 +1174,70 @@ impl DownloadQueue { self.finished.read().await.iter().find(|download| download.matches_existing_task(candidate)).cloned() } - /// Pause the active download. Persists the new state through the - /// transactional boundary. The runtime-only control signal is published - /// after the commit while the mutation guard still preserves ordering. - pub async fn pause_active(&self, uuid: &str) -> Result { + async fn transition_active_matching( + &self, + uuid: &str, + signal: DownloadControl, + mutate_fn: impl FnOnce(&mut PersistedFileDownload) -> bool, + ) -> Result { let _mutation = self.mutation_guard.lock().await; let changed = mutate_optional_locked(self, |candidate| { let Some(active) = candidate.active.as_mut().filter(|active| active.uuid == uuid) else { return Ok(None); }; - active.paused = true; - active.state = DownloadState::Paused; - active.next_retry_at = None; - Ok(Some(true)) + if mutate_fn(active) { + active.next_retry_at = None; + Ok(Some(true)) + } else { + Ok(None) + } }) .await? .unwrap_or(false); if !changed { return Ok(false); } - *self.control_signal.write().await = DownloadControl::Pause; + *self.control_signal.write().await = signal; self.control_notify.notify_waiters(); Ok(true) } + /// Pause the active download. Persists the new state through the + /// transactional boundary. The runtime-only control signal is published + /// after the commit while the mutation guard still preserves ordering. + pub async fn pause_active(&self, uuid: &str) -> Result { + self.transition_active_matching(uuid, DownloadControl::Pause, |active| { + active.paused = true; + active.state = DownloadState::Paused; + true + }) + .await + } + /// Resume the active download. Persists the new state through the /// transactional boundary. pub async fn resume_active(&self, uuid: &str) -> Result { - let _mutation = self.mutation_guard.lock().await; - let changed = mutate_optional_locked(self, |candidate| { - let Some(active) = candidate.active.as_mut().filter(|active| active.uuid == uuid && active.paused) else { - return Ok(None); - }; - active.paused = false; - active.state = DownloadState::Downloading; - active.next_retry_at = None; - Ok(Some(true)) + self.transition_active_matching(uuid, DownloadControl::None, |active| { + if active.paused { + active.paused = false; + active.state = DownloadState::Downloading; + true + } else { + false + } }) - .await? - .unwrap_or(false); - if !changed { - return Ok(false); - } - *self.control_signal.write().await = DownloadControl::None; - self.control_notify.notify_waiters(); - Ok(true) + .await } /// Cancel the active download. Persists the new state through the /// transactional boundary. pub async fn cancel_active_matching(&self, uuid: &str) -> Result { - let _mutation = self.mutation_guard.lock().await; - let changed = mutate_optional_locked(self, |candidate| { - let Some(active) = candidate.active.as_mut().filter(|active| active.uuid == uuid) else { - return Ok(None); - }; + self.transition_active_matching(uuid, DownloadControl::Cancel, |active| { active.state = DownloadState::Cancelled; active.error = Some("Cancelled by user".to_string()); - active.next_retry_at = None; - Ok(Some(true)) + true }) - .await? - .unwrap_or(false); - if !changed { - return Ok(false); - } - *self.control_signal.write().await = DownloadControl::Cancel; - self.control_notify.notify_waiters(); - Ok(true) + .await } pub async fn cancel_active(&self) -> Result { diff --git a/backend/dvr/src/recording/mod.rs b/backend/dvr/src/recording/mod.rs index 53efa6ad6..fbbc59478 100644 --- a/backend/dvr/src/recording/mod.rs +++ b/backend/dvr/src/recording/mod.rs @@ -33,3 +33,34 @@ pub use self::{ recording_security::*, recording_service::*, recording_source_resolution::*, recording_supervisor::*, recording_worker::*, recording_worker_runner::*, recording_ws::*, }; + +#[cfg(test)] +pub(crate) fn make_test_meta( + visibility: shared::model::recording::RecordingVisibility, + owner: shared::model::recording::RecordingOwner, + relative_path: Option<&str>, +) -> shared::model::RecordingMetadata { + shared::model::RecordingMetadata { + owner, + visibility, + source: None, + program_start: Some(1_700_000_000), + program_end: Some(1_700_003_600), + scheduled_start: Some(1_700_000_000), + scheduled_end: Some(1_700_003_600), + pre_roll_secs: 0, + post_roll_secs: 0, + channel_id: Some("ch-1".into()), + channel_name: Some("Channel 1".into()), + program_title: Some("Programme".into()), + epg: None, + provenance: shared::model::recording::RecordingProvenance::default(), + relative_path: relative_path.map(Into::into), + partial_relative_path: None, + reserved_bytes: 0, + measured_bytes: 0, + completed_at: None, + notification_markers: vec![], + deleting_previous_state: None, + } +} diff --git a/backend/dvr/src/recording/recording_conflict.rs b/backend/dvr/src/recording/recording_conflict.rs index d7d28cf5b..b26a5d05c 100644 --- a/backend/dvr/src/recording/recording_conflict.rs +++ b/backend/dvr/src/recording/recording_conflict.rs @@ -218,6 +218,16 @@ mod tests { DemandPoint { task_id: task_id.into(), padded_start: start, padded_end: end, priority } } + fn check_preview_severity( + candidate: &DemandPoint, + others: &[DemandPoint], + background_slots: u32, + reserved_interactive_slots: u32, + ) -> ConflictSeverity { + preview_conflict(candidate, others, EffectiveCapacity { background_slots, reserved_interactive_slots }, None) + .severity + } + #[test] fn empty_demand_means_no_known_conflict() { let candidate = cand(100, 200, 0); @@ -233,13 +243,8 @@ mod tests { // `other` has priority 1 — well below the candidate's 5. The // analyzer must ignore it. let others = vec![other("o1", 100, 200, 1)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 0 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::NoKnownConflict); + let sev = check_preview_severity(&candidate, &others, 1, 0); + assert_eq!(sev, ConflictSeverity::NoKnownConflict); } #[test] @@ -250,13 +255,8 @@ mod tests { // the whole window over → LikelyMissedWindow. let candidate = cand(100, 200, 0); let others = vec![other("o1", 100, 200, 0)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 0 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::LikelyMissedWindow); + let sev = check_preview_severity(&candidate, &others, 1, 0); + assert_eq!(sev, ConflictSeverity::LikelyMissedWindow); } #[test] @@ -266,13 +266,8 @@ mod tests { // 150..200 is under-capacity. Mixed → PossibleCapacityWait. let candidate = cand(100, 200, 0); let others = vec![other("o1", 100, 150, 0)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 0 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::PossibleCapacityWait); + let sev = check_preview_severity(&candidate, &others, 1, 0); + assert_eq!(sev, ConflictSeverity::PossibleCapacityWait); } #[test] @@ -280,13 +275,8 @@ mod tests { // 3 simultaneous demands on a 1-slot headroom. let candidate = cand(100, 200, 0); let others = vec![other("o1", 100, 200, 0), other("o2", 100, 200, 0), other("o3", 100, 200, 0)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 0 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::LikelyMissedWindow); + let sev = check_preview_severity(&candidate, &others, 1, 0); + assert_eq!(sev, ConflictSeverity::LikelyMissedWindow); } #[test] @@ -294,13 +284,8 @@ mod tests { // 1 background slot, 1 reserved interactive → headroom 0. let candidate = cand(100, 200, 0); let others = vec![other("o1", 100, 200, 0)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 1 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::LikelyMissedWindow); + let sev = check_preview_severity(&candidate, &others, 1, 1); + assert_eq!(sev, ConflictSeverity::LikelyMissedWindow); } #[test] @@ -310,26 +295,16 @@ mod tests { // under-capacity → mixed. let candidate = cand(100, 200, 0); let others = vec![other("o1", 150, 250, 0)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 0 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::PossibleCapacityWait); + let sev = check_preview_severity(&candidate, &others, 1, 0); + assert_eq!(sev, ConflictSeverity::PossibleCapacityWait); } #[test] fn no_overlap_means_no_known_conflict() { let candidate = cand(100, 200, 0); let others = vec![other("o1", 300, 400, 0)]; - let preview = preview_conflict( - &candidate, - &others, - EffectiveCapacity { background_slots: 1, reserved_interactive_slots: 0 }, - None, - ); - assert_eq!(preview.severity, ConflictSeverity::NoKnownConflict); + let sev = check_preview_severity(&candidate, &others, 1, 0); + assert_eq!(sev, ConflictSeverity::NoKnownConflict); } #[test] diff --git a/backend/dvr/src/recording/recording_notification.rs b/backend/dvr/src/recording/recording_notification.rs index 37c204c2c..f183ee2ec 100644 --- a/backend/dvr/src/recording/recording_notification.rs +++ b/backend/dvr/src/recording/recording_notification.rs @@ -143,30 +143,9 @@ mod tests { fn user(name: &str) -> UserId { UserId::from(name) } + #[inline] fn make_meta(visibility: RecordingVisibility, owner: RecordingOwner) -> RecordingMetadata { - RecordingMetadata { - owner, - visibility, - source: None, - program_start: Some(1_700_000_000), - program_end: Some(1_700_003_600), - scheduled_start: Some(1_700_000_000), - scheduled_end: Some(1_700_003_600), - pre_roll_secs: 0, - post_roll_secs: 0, - channel_id: Some("ch-1".into()), - channel_name: Some("Channel 1".into()), - program_title: Some("Programme".into()), - epg: None, - provenance: shared::model::recording::RecordingProvenance::default(), - relative_path: Some("users/web:alice/Programme_2023-11-14_20-00.ts".into()), - partial_relative_path: None, - reserved_bytes: 0, - measured_bytes: 0, - completed_at: None, - notification_markers: vec![], - deleting_previous_state: None, - } + crate::recording::make_test_meta(visibility, owner, Some("users/web:alice/Programme_2023-11-14_20-00.ts")) } #[test] diff --git a/backend/dvr/src/recording/recording_notification_adapter.rs b/backend/dvr/src/recording/recording_notification_adapter.rs index 993d6e025..1b6583032 100644 --- a/backend/dvr/src/recording/recording_notification_adapter.rs +++ b/backend/dvr/src/recording/recording_notification_adapter.rs @@ -104,30 +104,9 @@ mod tests { fn user(name: &str) -> UserId { UserId::from(name) } + #[inline] fn make_meta(visibility: RecordingVisibility, owner: RecordingOwner) -> RecordingMetadata { - RecordingMetadata { - owner, - visibility, - source: None, - program_start: Some(1_700_000_000), - program_end: Some(1_700_003_600), - scheduled_start: Some(1_700_000_000), - scheduled_end: Some(1_700_003_600), - pre_roll_secs: 0, - post_roll_secs: 0, - channel_id: Some("ch-1".into()), - channel_name: Some("Channel 1".into()), - program_title: Some("Programme".into()), - epg: None, - provenance: shared::model::recording::RecordingProvenance::default(), - relative_path: Some("path/file.ts".into()), - partial_relative_path: None, - reserved_bytes: 0, - measured_bytes: 0, - completed_at: None, - notification_markers: vec![], - deleting_previous_state: None, - } + crate::recording::make_test_meta(visibility, owner, Some("path/file.ts")) } #[test] diff --git a/backend/dvr/src/recording/recording_security.rs b/backend/dvr/src/recording/recording_security.rs index ae85fcae2..7139037e2 100644 --- a/backend/dvr/src/recording/recording_security.rs +++ b/backend/dvr/src/recording/recording_security.rs @@ -191,30 +191,9 @@ mod tests { fn user(name: &str) -> UserId { UserId::from(name) } + #[inline] fn make_meta(visibility: RecordingVisibility, owner: RecordingOwner) -> RecordingMetadata { - RecordingMetadata { - owner, - visibility, - source: None, - program_start: Some(1_700_000_000), - program_end: Some(1_700_003_600), - scheduled_start: Some(1_700_000_000), - scheduled_end: Some(1_700_003_600), - pre_roll_secs: 0, - post_roll_secs: 0, - channel_id: Some("ch-1".into()), - channel_name: Some("Channel 1".into()), - program_title: Some("Programme".into()), - epg: None, - provenance: shared::model::recording::RecordingProvenance::default(), - relative_path: Some("path/file.ts".into()), - partial_relative_path: None, - reserved_bytes: 0, - measured_bytes: 0, - completed_at: None, - notification_markers: vec![], - deleting_previous_state: None, - } + crate::recording::make_test_meta(visibility, owner, Some("path/file.ts")) } #[test] diff --git a/backend/hls/src/manager.rs b/backend/hls/src/manager.rs index 1caf74a49..21bcd5c1c 100644 --- a/backend/hls/src/manager.rs +++ b/backend/hls/src/manager.rs @@ -3770,6 +3770,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::similar_names, clippy::too_many_lines)] async fn finalized_signed_url_generations_follow_published_lease_lifetime() { let manager = HlsProxyManager::new(); let published_resource_ids = HlsPublishedTransientResourceIds::default(); @@ -3914,6 +3915,7 @@ mod tests { } #[tokio::test] + #[allow(clippy::similar_names, clippy::too_many_lines)] async fn same_lease_retains_every_published_finalized_generation_until_removal() { let manager = HlsProxyManager::new(); let published_resource_ids = HlsPublishedTransientResourceIds::default(); diff --git a/backend/hls/src/manifest_limits.rs b/backend/hls/src/manifest_limits.rs index d5d2ceb12..9a6d094f1 100644 --- a/backend/hls/src/manifest_limits.rs +++ b/backend/hls/src/manifest_limits.rs @@ -81,7 +81,7 @@ mod tests { #[test] fn finalized_catchup_example_fits_representation_limits() { - assert!(MAX_TRANSIENT_MANIFEST_RESOURCES >= 1_643); - assert!(MAX_HLS_LEASE_SNAPSHOT_SEGMENTS >= 1_643); + const { assert!(MAX_TRANSIENT_MANIFEST_RESOURCES >= 1_643) }; + const { assert!(MAX_HLS_LEASE_SNAPSHOT_SEGMENTS >= 1_643) }; } } diff --git a/backend/hls/src/refresh/tests.rs b/backend/hls/src/refresh/tests.rs index 2270ffc50..7f33a4d43 100644 --- a/backend/hls/src/refresh/tests.rs +++ b/backend/hls/src/refresh/tests.rs @@ -86,6 +86,7 @@ use shared::model::{ ProviderUrlSelectionPolicy, }; use std::{ + fmt::Write, io, sync::{ atomic::{AtomicUsize, Ordering}, @@ -2092,7 +2093,7 @@ async fn finalized_event_manifest_commits_full_transient_lifecycle() { let mut origin_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); for sequence in 1..=8 { - origin_body.push_str(&format!("#EXTINF:4.0,\n{sequence}.ts\n")); + let _ = write!(origin_body, "#EXTINF:4.0,\n{sequence}.ts\n"); } origin_body.push_str("#EXT-X-ENDLIST\n"); let origin_body: &'static str = Box::leak(origin_body.into_boxed_str()); @@ -2124,7 +2125,7 @@ async fn finalized_event_manifest_commits_full_transient_lifecycle() { async fn endlist_only_manifest_commits_complete_body_with_consistent_finalized_state() { let mut origin_body = "#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXT-X-MEDIA-SEQUENCE:1\n".to_string(); for sequence in 1..=8 { - origin_body.push_str(&format!("#EXTINF:4.0,\n{sequence}.ts\n")); + let _ = write!(origin_body, "#EXTINF:4.0,\n{sequence}.ts\n"); } origin_body.push_str("#EXT-X-ENDLIST\n"); let origin_body: &'static str = Box::leak(origin_body.into_boxed_str()); diff --git a/backend/hls/src/transient.rs b/backend/hls/src/transient.rs index 9d3babdc8..18ee34533 100644 --- a/backend/hls/src/transient.rs +++ b/backend/hls/src/transient.rs @@ -2350,7 +2350,7 @@ mod tests { let resources = base.resources; let generations_within_limit = MAX_TRANSIENT_GENERATION_MEMBERSHIPS / MAX_TRANSIENT_MANIFEST_RESOURCES; for index in 0..generations_within_limit { - body.push_str(&format!("# generation-{index}\n")); + let _ = writeln!(body, "# generation-{index}"); state .commit_rewritten_manifest_with_semantics( body.clone(), @@ -2995,14 +2995,10 @@ mod tests { let proxy_session_id = ProxySessionId("proxy-session".to_string()); let mut origin_body = String::from("#EXTM3U\n#EXT-X-TARGETDURATION:7\n#EXT-X-PLAYLIST-TYPE:EVENT\n"); let durations_ms = (0..46) - .map(|index| { - if index < 44 { - 6_700 - } else if index == 44 { - 6_720 - } else { - 6_000 - } + .map(|index| match index { + 0..44 => 6_700, + 44 => 6_720, + _ => 6_000, }) .collect::>(); for (index, duration_ms) in durations_ms.iter().enumerate() { diff --git a/backend/iptv/src/capability_store.rs b/backend/iptv/src/capability_store.rs index 2e4869710..eec8fae5b 100644 --- a/backend/iptv/src/capability_store.rs +++ b/backend/iptv/src/capability_store.rs @@ -110,15 +110,8 @@ async fn read_snapshot(path: &Path) -> ProviderCapabilities { } } -fn sanitize_component(value: &str) -> String { - value - .chars() - .map(|ch| match ch { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' => ch, - _ => '_', - }) - .collect() -} +#[inline] +fn sanitize_component(value: &str) -> String { crate::redaction::sanitize_path_component(value, false) } #[cfg(test)] mod tests { diff --git a/backend/iptv/src/redaction.rs b/backend/iptv/src/redaction.rs index bad0dc760..5972a2fcb 100644 --- a/backend/iptv/src/redaction.rs +++ b/backend/iptv/src/redaction.rs @@ -141,9 +141,31 @@ pub fn redact_json(value: &mut serde_json::Value) { } } +/// Sanitize a string for safe use in file/directory path components. +/// +/// If `allow_dots` is true, '.' characters are preserved (e.g. for hostnames); +/// otherwise they are replaced with '_'. +#[must_use] +pub fn sanitize_path_component(value: &str, allow_dots: bool) -> String { + value + .chars() + .map(|ch| match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' => ch, + '.' if allow_dots => '.', + _ => '_', + }) + .collect() +} + #[cfg(test)] mod tests { - use super::{is_sensitive_key, redact_json, redact_text, safe_url}; + use super::{is_sensitive_key, redact_json, redact_text, safe_url, sanitize_path_component}; + + #[test] + fn sanitize_path_component_preserves_allowed_chars() { + assert_eq!(sanitize_path_component("abc-XYZ_123.host/extra", true), "abc-XYZ_123.host_extra"); + assert_eq!(sanitize_path_component("abc-XYZ_123.host/extra", false), "abc-XYZ_123_host_extra"); + } #[test] fn safe_url_drops_userinfo_query_and_fragment() { diff --git a/backend/iptv/src/stalker/client.rs b/backend/iptv/src/stalker/client.rs index 7fbc0552f..b55256ab7 100644 --- a/backend/iptv/src/stalker/client.rs +++ b/backend/iptv/src/stalker/client.rs @@ -834,15 +834,8 @@ fn rotate_stalker_debug_dumps(dump_dir: &Path) { } } -fn sanitize_dump_component(value: &str) -> String { - value - .chars() - .map(|ch| match ch { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => ch, - _ => '_', - }) - .collect() -} +#[inline] +fn sanitize_dump_component(value: &str) -> String { crate::redaction::sanitize_path_component(value, true) } #[cfg(test)] mod transport_tests { diff --git a/backend/iptv/src/stalker/parser.rs b/backend/iptv/src/stalker/parser.rs index 7ae2e8b83..54a8f2d61 100644 --- a/backend/iptv/src/stalker/parser.rs +++ b/backend/iptv/src/stalker/parser.rs @@ -293,7 +293,7 @@ fn map_stalker_episode( let season_number = episode.season_number.or(season.number).unwrap_or(0); let cmd = episode.cmd.clone().unwrap_or_default(); let info = episode.info.clone(); - let container_extension = episode.container_extension.clone().filter(|s| !s.is_empty()); + let container_extension = episode.container_extension.clone().filter(|s| !shared::utils::is_nullish(s.as_str())); let logo = info.as_ref().and_then(|i| i.movie_image.clone()).filter(|s| !s.is_empty()); let descriptor = if cmd.is_empty() { diff --git a/backend/media-server/Cargo.toml b/backend/media-server/Cargo.toml index 0a511940a..8789f762d 100644 --- a/backend/media-server/Cargo.toml +++ b/backend/media-server/Cargo.toml @@ -25,4 +25,4 @@ url.workspace = true # against; hyper is not used by the module itself. http-body-util = "0.1.5" hyper = "1.11.0" -hyper-util = "0.1.20" +hyper-util = { version = "0.1.20", features = ["server", "server-auto", "tokio"] } diff --git a/backend/media-server/src/playback.rs b/backend/media-server/src/playback.rs index dd2684278..4581d36a7 100644 --- a/backend/media-server/src/playback.rs +++ b/backend/media-server/src/playback.rs @@ -8,7 +8,10 @@ use reqwest::{ header::{HeaderMap, ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, LAST_MODIFIED}, StatusCode, }; -use shared::model::{InputType, PlaylistItemType}; +use shared::{ + model::{InputType, PlaylistItemType}, + utils::hex_digit, +}; use std::{fmt, sync::Arc}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -217,16 +220,7 @@ fn unescape_internal_url_component(value: &str) -> String { String::from_utf8_lossy(&decoded).into_owned() } -fn decode_hex_byte(high: u8, low: u8) -> Option { Some(hex_value(high)? << 4 | hex_value(low)?) } - -fn hex_value(value: u8) -> Option { - match value { - b'0'..=b'9' => Some(value - b'0'), - b'a'..=b'f' => Some(value - b'a' + 10), - b'A'..=b'F' => Some(value - b'A' + 10), - _ => None, - } -} +fn decode_hex_byte(high: u8, low: u8) -> Option { Some(hex_digit(high)? << 4 | hex_digit(low)?) } #[cfg(test)] mod tests { diff --git a/backend/media-server/src/plex/client.rs b/backend/media-server/src/plex/client.rs index e0457c56c..b2e4aa9ef 100644 --- a/backend/media-server/src/plex/client.rs +++ b/backend/media-server/src/plex/client.rs @@ -1037,37 +1037,42 @@ mod tests { assert_eq!(libraries[0].reference.library_id.as_ref(), "2"); } - #[test] - fn pms_part_url_accepts_only_same_origin_part_resources() { + fn assert_pms_resource_url_guards( + resolve: fn(&StdArc, &str) -> Result, + valid_path: &str, + expected_valid_url: &str, + wrong_prefix_path: &str, + expected_err_kind: MediaServerErrorKind, + ) { let base = StdArc::::from("http://127.0.0.1:32400/base"); - + assert_eq!(resolve(&base, valid_path).expect("resource key resolves"), expected_valid_url); + assert_eq!(resolve(&base, wrong_prefix_path).expect_err("wrong prefix must fail").kind, expected_err_kind); assert_eq!( - pms_part_url(&base, "/library/parts/part-redacted/file.mkv?download=1").expect("part key resolves"), - "http://127.0.0.1:32400/library/parts/part-redacted/file.mkv?download=1" - ); - assert_eq!( - pms_part_url(&base, "/library/metadata/rating-redacted") - .expect_err("metadata paths are not direct part refs") - .kind, - MediaServerErrorKind::NoDirectPlayableMediaServerSource - ); - assert_eq!( - pms_part_url(&base, "//evil.example.invalid/library/parts/part-redacted/file.mkv") + resolve(&base, &format!("//evil.example.invalid{valid_path}")) .expect_err("network-path refs must not escape the selected PMS") .kind, - MediaServerErrorKind::NoDirectPlayableMediaServerSource + expected_err_kind ); assert_eq!( - pms_part_url(&base, "/library/parts/../../identity") - .expect_err("normalized paths must stay under direct part refs") - .kind, - MediaServerErrorKind::NoDirectPlayableMediaServerSource + resolve(&base, "/library/../identity").expect_err("normalized paths must stay under direct refs").kind, + expected_err_kind ); assert_eq!( - pms_part_url(&base, "/library/parts/part-redacted/file.mkv?X-Plex-Token=should-not-leak") - .expect_err("part refs must not carry credentials") + resolve(&base, &format!("{valid_path}&X-Plex-Token=should-not-leak")) + .expect_err("refs must not carry credentials") .kind, - MediaServerErrorKind::NoDirectPlayableMediaServerSource + expected_err_kind + ); + } + + #[test] + fn pms_part_url_accepts_only_same_origin_part_resources() { + assert_pms_resource_url_guards( + pms_part_url, + "/library/parts/part-redacted/file.mkv?download=1", + "http://127.0.0.1:32400/library/parts/part-redacted/file.mkv?download=1", + "/library/metadata/rating-redacted", + MediaServerErrorKind::NoDirectPlayableMediaServerSource, ); } @@ -1077,40 +1082,18 @@ mod tests { /// in the shared `resolve_pms_resource_url` helper. #[test] fn pms_image_url_accepts_only_same_origin_metadata_images() { - let base = StdArc::::from("http://127.0.0.1:32400/base"); - - assert_eq!( - pms_image_url(&base, "/library/metadata/rating-redacted/thumb?width=320").expect("image path resolves"), - "http://127.0.0.1:32400/library/metadata/rating-redacted/thumb?width=320" + assert_pms_resource_url_guards( + pms_image_url, + "/library/metadata/rating-redacted/thumb?width=320", + "http://127.0.0.1:32400/library/metadata/rating-redacted/thumb?width=320", + "/library/parts/part-redacted/file.mkv", + MediaServerErrorKind::MediaServerItemNotFound, ); + let base = StdArc::::from("http://127.0.0.1:32400/base"); assert_eq!( pms_image_url(&base, "").expect_err("blank paths must be rejected").kind, MediaServerErrorKind::MediaServerItemNotFound ); - assert_eq!( - pms_image_url(&base, "/library/parts/part-redacted/file.mkv") - .expect_err("part paths are not metadata images") - .kind, - MediaServerErrorKind::MediaServerItemNotFound - ); - assert_eq!( - pms_image_url(&base, "//evil.example.invalid/library/metadata/rating-redacted/thumb") - .expect_err("network-path refs must not escape the selected PMS") - .kind, - MediaServerErrorKind::MediaServerItemNotFound - ); - assert_eq!( - pms_image_url(&base, "/library/metadata/../../identity") - .expect_err("normalized paths must stay under metadata images") - .kind, - MediaServerErrorKind::MediaServerItemNotFound - ); - assert_eq!( - pms_image_url(&base, "/library/metadata/rating-redacted/thumb?X-Plex-Token=should-not-leak") - .expect_err("image refs must not carry credentials") - .kind, - MediaServerErrorKind::MediaServerItemNotFound - ); } fn plex_test_response(status: StatusCode, body: &'static [u8]) -> Response> { diff --git a/backend/parser/src/hls/mod.rs b/backend/parser/src/hls/mod.rs index fcb923865..a96de68af 100644 --- a/backend/parser/src/hls/mod.rs +++ b/backend/parser/src/hls/mod.rs @@ -223,30 +223,15 @@ mod test { } #[test] - fn rewrite_http_relative_segment() { - let base = "http://example.com/hls/playlist.m3u8"; - let uri = "seg001.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "http://example.com/hls/seg001.ts"); - } - - #[test] - fn rewrite_http_root_relative_segment() { - let base = "http://example.com/hls/playlist.m3u8"; - let uri = "/media/seg001.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "http://example.com/media/seg001.ts"); - } - - #[test] - fn rewrite_http_parent_directory() { - let base = "http://example.com/hls/level1/playlist.m3u8"; - let uri = "../seg001.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "http://example.com/hls/seg001.ts"); + fn rewrite_http_relative_segments() { + let cases = [ + ("http://example.com/hls/playlist.m3u8", "seg001.ts", "http://example.com/hls/seg001.ts"), + ("http://example.com/hls/playlist.m3u8", "/media/seg001.ts", "http://example.com/media/seg001.ts"), + ("http://example.com/hls/level1/playlist.m3u8", "../seg001.ts", "http://example.com/hls/seg001.ts"), + ]; + for (base, uri, expected) in cases { + assert_eq!(rewrite_hls_url(base, uri), expected); + } } #[test] @@ -288,55 +273,62 @@ mod test { } #[test] - fn rewrite_relative_variant_preserves_archive_start_query() { - let base = "https://cdn.example/hls/channel/index.m3u8?offset=-10752&utcstart=1785072000&useseq=t"; - let uri = "variant/playlist.m3u8?offset=-10752&useseq=t"; + fn rewrite_hls_url_cases() { + let cases = [ + ( + "https://cdn.example/hls/channel/index.m3u8?offset=-10752&utcstart=1785072000&useseq=t", + "variant/playlist.m3u8?offset=-10752&useseq=t", + "https://cdn.example/hls/channel/variant/playlist.m3u8?offset=-10752&useseq=t&utcstart=1785072000", + ), + ( + "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600&end=1785075600&duration=3600", + "variant/playlist.m3u8?offset=-1800", + "https://cdn.example/hls/channel/variant/playlist.m3u8?offset=-1800&utcstart=1785072000&end=1785075600&duration=3600", + ), + ( + "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000", + "variant/playlist.m3u8?utc=1785071000", + "https://cdn.example/hls/channel/variant/playlist.m3u8?utc=1785071000", + ), + ( + "https://cdn.example/hls/channel/index.m3u8?start=1785072000", + "variant/playlist.m3u8", + "https://cdn.example/hls/channel/variant/playlist.m3u8", + ), + ( + "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600", + "segment.ts?sig=abc", + "https://cdn.example/hls/channel/segment.ts?sig=abc", + ), + ( + "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600", + "key.bin?sig=def", + "https://cdn.example/hls/channel/key.bin?sig=def", + ), + ( + "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600", + "init.mp4?sig=ghi", + "https://cdn.example/hls/channel/init.mp4?sig=ghi", + ), + ( + "http://example.com/hls/playlist.m3u8", + "https://cdn.example.org/video/seg.ts", + "https://cdn.example.org/video/seg.ts", + ), + ("file:///mnt/media/hls/playlist.m3u8", "seg001.ts", "file:///mnt/media/hls/seg001.ts"), + ("file:///mnt/media/hls/level1/playlist.m3u8", "../seg001.ts", "file:///mnt/media/hls/seg001.ts"), + ( + "file:///mnt/media/hls/playlist.m3u8?utc=1785072000", + "child.m3u8", + "file:///mnt/media/hls/child.m3u8", + ), + ("file:///mnt/media/hls/playlist.m3u8", "file:///mnt/other/seg.ts", "file:///mnt/other/seg.ts"), + ("http://example.com/hls/playlist.m3u8", "seg.ts#t=10", "http://example.com/hls/seg.ts#t=10"), + ]; - let out = rewrite_hls_url(base, uri); - assert_eq!( - out, - "https://cdn.example/hls/channel/variant/playlist.m3u8?offset=-10752&useseq=t&utcstart=1785072000" - ); - } - - #[test] - fn rewrite_relative_variant_preserves_all_archive_context_queries() { - let base = - "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600&end=1785075600&duration=3600"; - let uri = "variant/playlist.m3u8?offset=-1800"; - - let out = rewrite_hls_url(base, uri); - assert_eq!( - out, - "https://cdn.example/hls/channel/variant/playlist.m3u8?offset=-1800&utcstart=1785072000&end=1785075600&duration=3600" - ); - } - - #[test] - fn rewrite_keeps_child_archive_start_query() { - let base = "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000"; - let uri = "variant/playlist.m3u8?utc=1785071000"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "https://cdn.example/hls/channel/variant/playlist.m3u8?utc=1785071000"); - } - - #[test] - fn rewrite_does_not_propagate_plain_start_query() { - let base = "https://cdn.example/hls/channel/index.m3u8?start=1785072000"; - let uri = "variant/playlist.m3u8"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "https://cdn.example/hls/channel/variant/playlist.m3u8"); - } - - #[test] - fn rewrite_archive_playlist_does_not_modify_signed_media_urls() { - let base = "https://cdn.example/hls/channel/index.m3u8?utcstart=1785072000&offset=-3600"; - - assert_eq!(rewrite_hls_url(base, "segment.ts?sig=abc"), "https://cdn.example/hls/channel/segment.ts?sig=abc"); - assert_eq!(rewrite_hls_url(base, "key.bin?sig=def"), "https://cdn.example/hls/channel/key.bin?sig=def"); - assert_eq!(rewrite_hls_url(base, "init.mp4?sig=ghi"), "https://cdn.example/hls/channel/init.mp4?sig=ghi"); + for (base, uri, expected) in cases { + assert_eq!(rewrite_hls_url(base, uri), expected, "failed for base: {base}, uri: {uri}"); + } } #[test] @@ -348,58 +340,6 @@ mod test { assert_eq!(rewrite_hls_url(base, reference), reference); } - #[test] - fn rewrite_https_absolute_passthrough() { - let base = "http://example.com/hls/playlist.m3u8"; - let uri = "https://cdn.example.org/video/seg.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, uri); - } - - #[test] - fn rewrite_file_relative_segment() { - let base = "file:///mnt/media/hls/playlist.m3u8"; - let uri = "seg001.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "file:///mnt/media/hls/seg001.ts"); - } - - #[test] - fn rewrite_file_parent_directory() { - let base = "file:///mnt/media/hls/level1/playlist.m3u8"; - let uri = "../seg001.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, "file:///mnt/media/hls/seg001.ts"); - } - - #[test] - fn rewrite_file_child_playlist_does_not_inherit_archive_query() { - let base = "file:///mnt/media/hls/playlist.m3u8?utc=1785072000"; - - assert_eq!(rewrite_hls_url(base, "child.m3u8"), "file:///mnt/media/hls/child.m3u8"); - } - - #[test] - fn rewrite_file_absolute_passthrough() { - let base = "file:///mnt/media/hls/playlist.m3u8"; - let uri = "file:///mnt/other/seg.ts"; - - let out = rewrite_hls_url(base, uri); - assert_eq!(out, uri); - } - - #[test] - fn rewrite_hls_fragment() { - let base = "http://example.com/hls/playlist.m3u8"; - let fragment = "seg.ts#t=10"; - - let out = rewrite_hls_url(base, fragment); - assert_eq!(out, "http://example.com/hls/seg.ts#t=10"); - } - #[test] fn rewrite_hls_without_user_token_keeps_segment_urls() { let mut user = ProxyUserCredentials::default(); diff --git a/backend/parser/src/xtream.rs b/backend/parser/src/xtream.rs index 7a0aa4355..dcc480674 100644 --- a/backend/parser/src/xtream.rs +++ b/backend/parser/src/xtream.rs @@ -845,4 +845,17 @@ mod tests { assert_eq!(groups[1].channels[0].header.name.as_ref(), "unknown-1"); assert_eq!(groups[1].channels[0].header.source_ordinal, 2); } + + #[test] + fn xtream_episode_collapses_literal_null_container_extension_to_empty() { + let parsed: SeriesStreamDetailEpisodeProperties = serde_json::from_str( + r#"{"id":101,"episode_num":1,"season":1,"title":"S01E01","container_extension":"null"}"#, + ) + .unwrap(); + assert!( + parsed.container_extension.is_empty(), + "literal \"null\" must not survive as an extension, got {:?}", + parsed.container_extension + ); + } } diff --git a/backend/processing/src/processor/epg.rs b/backend/processing/src/processor/epg.rs index 48b5234c2..a4dc7b658 100644 --- a/backend/processing/src/processor/epg.rs +++ b/backend/processing/src/processor/epg.rs @@ -2,7 +2,7 @@ use crate::{fetched_playlist::FetchedPlaylist, parser::xmltv::normalize_channel_ use log::{debug, trace, warn}; use rphonetic::{DoubleMetaphone, Encoder}; use shared::{ - model::{EpgNamePrefix, EpgSmartMatchConfigDto, PlaylistItem, XtreamCluster}, + model::{EpgNamePrefix, EpgSmartMatchConfigDto, PlaylistGroup, PlaylistItem, XtreamCluster}, utils::{Internable, CONSTANTS}, }; use std::{ @@ -743,13 +743,16 @@ fn assign_live_channel_epg( icon_override_channels: &HashSet>, icon_assigned: &mut HashSet>, stats: &mut EpgAssignmentStats, -) -> bool { + clear_invalid_epg_ids: bool, +) { if id_cache.smart_match_enabled { stats.record(assign_smart_epg_id(channel, id_cache)); } let has_epg = has_processed_epg(channel, id_cache); assign_epg_icon(channel, icon_tags, icon_override_channels, icon_assigned); - has_epg + if clear_invalid_epg_ids && !has_epg { + channel.header.epg_channel_id = None; + } } fn referenced_live_epg_ids(fp: &mut FetchedPlaylist<'_>) -> HashSet> { @@ -761,13 +764,27 @@ fn referenced_live_epg_ids(fp: &mut FetchedPlaylist<'_>) -> HashSet> { .collect() } -pub(crate) fn retain_live_items_with_processed_epg(fp: &mut FetchedPlaylist<'_>, epg: &[Epg]) { +pub(crate) fn retain_epg_referenced_by_groups(groups: &[PlaylistGroup], epg: &mut [Epg]) { + let referenced_epg_ids = groups + .iter() + .flat_map(|group| &group.channels) + .filter(|channel| is_live_epg_item(channel)) + .filter_map(|channel| { + channel.header.epg_channel_id.as_ref().map(|id| with_folded_epg_id(id, |folded| folded.intern())) + }) + .collect::>(); + for source in epg { + source.children.retain(|channel| with_folded_epg_id(&channel.id, |folded| referenced_epg_ids.contains(folded))); + } +} + +pub(crate) fn clear_invalid_live_epg_ids(fp: &mut FetchedPlaylist<'_>, epg: &[Epg]) { let processed_epg_ids = epg .iter() .flat_map(|source| &source.children) .map(|channel| with_folded_epg_id(&channel.id, |folded| folded.intern())) .collect::>(); - let mut removed = 0usize; + let mut cleared = 0usize; fp.source.retain_memory_items_mut(|item| { if !is_live_epg_item(item) { return true; @@ -777,18 +794,18 @@ pub(crate) fn retain_live_items_with_processed_epg(fp: &mut FetchedPlaylist<'_>, .epg_channel_id .as_deref() .is_some_and(|id| with_folded_epg_id(id, |folded| processed_epg_ids.contains(folded))); - if !has_epg { - removed += 1; + if !has_epg && item.header.epg_channel_id.take().is_some() { + cleared += 1; } - has_epg + true }); - if removed > 0 { - debug!("Removed {removed} live channels invalidated by after-EPG mappings from input '{}'", fp.input.name); + if cleared > 0 { + debug!("Cleared {cleared} unmatched EPG IDs from input '{}'", fp.input.name); } } /// Assigns EPG IDs and logos to live playlist channels by matching them with EPG data. -/// When EPG data is required by the target, unmatched live entries are removed in the same pass. +/// Invalid EPG IDs are optionally cleared without removing playlist entries. /// /// For each live channel in the playlist missing an EPG ID, attempts to assign one using normalized name matching if smart matching is enabled. If a channel has an EPG ID but lacks logos, assigns logos from the corresponding EPG icon tags. Adds the matched EPG data to the provided vector. /// @@ -804,18 +821,14 @@ async fn assign_channel_epg( new_epg: &mut Vec, fp: &mut FetchedPlaylist<'_>, id_cache: &mut EpgIdCache, - required_epg: bool, + clear_invalid_epg_ids: bool, ) { let Some(tv_guide) = &fp.epg else { return; }; let mut merged_epg = tv_guide.filter_merged_with_icon_overrides(id_cache).await; - if merged_epg.is_none() && !required_epg { - return; - } let mut stats = EpgAssignmentStats::default(); - let mut removed = 0usize; if fp.is_memory() { let icon_tags = merged_epg .as_ref() @@ -838,30 +851,20 @@ async fn assign_channel_epg( let mut process_channel = |channel: &mut PlaylistItem| { if !is_live_epg_item(channel) { - return true; + return; } - let has_epg = assign_live_channel_epg( + assign_live_channel_epg( channel, id_cache, &icon_tags, &icon_override_channels, &mut icon_assigned, &mut stats, + clear_invalid_epg_ids, ); - if required_epg && !has_epg { - removed += 1; - return false; - } - true }; - if required_epg { - fp.source.retain_memory_items_mut(&mut process_channel); - } else { - fp.items_mut().for_each(|channel| { - process_channel(channel); - }); - } + fp.items_mut().for_each(&mut process_channel); } else { warn!("Disk based playlist modification is not supported!"); } @@ -872,10 +875,6 @@ async fn assign_channel_epg( fp.input.name, stats.live, stats.existing, stats.exact, stats.fuzzy, stats.corrected, stats.unresolved ); } - if removed > 0 { - debug!("Removed {removed} live channels without EPG from input '{}'", fp.input.name); - } - if let Some((mut epg_source, _)) = merged_epg.take() { let referenced_epg_ids = referenced_live_epg_ids(fp); epg_source @@ -896,7 +895,7 @@ async fn assign_channel_epg( /// let mut epg_data = Vec::new(); /// process_playlist_epg(&mut playlist, &mut epg_data, false); /// ``` -pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec, required_epg: bool) { +pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec, clear_invalid_epg_ids: bool) { if fp.input.epg.is_none() { return; } @@ -904,10 +903,10 @@ pub async fn process_playlist_epg(fp: &mut FetchedPlaylist<'_>, epg: &mut Vec)], smart_matching: bool, - required_epg: bool, + clear_invalid_epg_ids: bool, ) -> (Vec>>, Vec) { let dir = tempdir().unwrap(); let epg_path = dir.path().join("smart-match.xml"); @@ -1045,7 +1048,7 @@ mod tests { }; let mut epg = Vec::new(); - super::process_playlist_epg(&mut playlist, &mut epg, required_epg).await; + super::process_playlist_epg(&mut playlist, &mut epg, clear_invalid_epg_ids).await; let assigned_ids = playlist.items_mut().map(|item| item.header.epg_channel_id.clone()).collect(); (assigned_ids, epg) } @@ -1270,7 +1273,7 @@ mod tests { } #[test] - fn required_epg_removes_only_unmatched_live_items() { + fn clear_invalid_epg_ids_preserves_items_and_clears_only_unmatched_live_ids() { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async move { let dir = tempdir().unwrap(); @@ -1334,18 +1337,42 @@ mod tests { super::process_playlist_epg(&mut playlist, &mut epg, true).await; - let names = playlist.items_mut().map(|item| item.header.name.clone()).collect::>(); - assert!(names.contains("Matched Live")); - assert!(!names.contains("Unmatched Live")); - assert!(names.contains("VOD")); - assert!(names.contains("Series")); - assert!(names.contains("Local VOD")); - assert!(names.contains("Local Series")); + let items = playlist + .items_mut() + .map(|item| (item.header.name.clone(), item.header.epg_channel_id.clone())) + .collect::>(); + assert_eq!(items.get("Matched Live").and_then(Option::as_deref), Some("matched.live")); + assert_eq!(items.get("Unmatched Live"), Some(&None)); + assert!(items.contains_key("VOD")); + assert!(items.contains_key("Series")); + assert!(items.contains_key("Local VOD")); + assert!(items.contains_key("Local Series")); }); } #[test] - fn required_epg_is_ignored_without_a_materialized_epg_source() { + fn default_epg_processing_preserves_unmatched_existing_id() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async move { + let (assigned_ids, _) = run_xmltv_matches( + r#" + Matched Live + + Programme + +"#, + &[("Unmatched Live", Some("missing.live"))], + false, + false, + ) + .await; + + assert_eq!(assigned_ids, vec![Some("missing.live".intern())]); + }); + } + + #[test] + fn clear_invalid_epg_ids_is_ignored_without_a_materialized_epg_source() { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async move { let mut input = ConfigInput::from(ConfigInputDto::default()); @@ -1362,6 +1389,10 @@ mod tests { super::process_playlist_epg(&mut playlist, &mut Vec::new(), true).await; assert_eq!(playlist.items_mut().count(), 1); + assert_eq!( + playlist.items_mut().next().and_then(|item| item.header.epg_channel_id.as_deref()), + Some("missing.live") + ); }); } @@ -1417,7 +1448,7 @@ mod tests { } #[test] - fn required_epg_removes_live_items_without_ids_when_smart_matching_is_disabled() { + fn clear_invalid_epg_ids_preserves_live_items_without_ids() { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async move { let dir = tempdir().unwrap(); @@ -1446,13 +1477,13 @@ mod tests { super::process_playlist_epg(&mut playlist, &mut Vec::new(), true).await; - assert_eq!(playlist.items_mut().count(), 0); - assert_eq!(playlist.get_group_count(), 0); + assert_eq!(playlist.items_mut().count(), 1); + assert_eq!(playlist.get_group_count(), 1); }); } #[test] - fn required_epg_keeps_live_items_assigned_by_smart_matching() { + fn clear_invalid_epg_ids_keeps_ids_assigned_by_smart_matching() { let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.block_on(async move { let (assigned_ids, epg) = run_xmltv_matches( diff --git a/backend/processing/src/processor/playlist.rs b/backend/processing/src/processor/playlist.rs index 820570e52..656caf76d 100644 --- a/backend/processing/src/processor/playlist.rs +++ b/backend/processing/src/processor/playlist.rs @@ -7,7 +7,7 @@ use crate::{ parser::xmltv::{flatten_tvguide, merge_epg_trees, EpgMergeAccumulator, TVGuide}, playlist_watch::{process_group_watch, process_target_groups_watch}, processor::{ - epg::{process_playlist_epg, retain_live_items_with_processed_epg}, + epg::{clear_invalid_live_epg_ids, process_playlist_epg, retain_epg_referenced_by_groups}, sort::sort_playlist, trakt::process_trakt_categories_for_target, xtream_series::playlist_resolve_series, @@ -45,9 +45,10 @@ use tokio::{ }; use tuliprox_core::{ model::{ - is_valid, AppConfig, CompiledMapping, ConfigFavourites, ConfigInput, ConfigInputFlags, ConfigInputOptions, - ConfigRename, ConfigTarget, Epg, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, - ReverseProxyDisabledHeaderConfig, TransformStage, UpdateGuard, UpdateTask, + is_valid, retain_filtered_playlist, AppConfig, CompiledMapping, ConfigFavourites, ConfigInput, + ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, Epg, FilterOutcome, MappingProgram, + ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig, TransformStage, UpdateGuard, + UpdateTask, }, utils::{debug_if_enabled, log_memory_snapshot, trace_if_enabled, StepMeasure, StepMeasureCallback}, }; @@ -116,13 +117,6 @@ fn stalker_checkpoint_message(input: &str) -> String { format!("Input '{input}': Stalker refresh checkpoint saved; active snapshot remains in service") } -#[derive(Debug, Default, PartialEq, Eq)] -pub struct FilterOutcome { - pub inspected: usize, - pub retained: usize, - pub removed: usize, -} - fn retain_playlist_items( source: &mut PlaylistSource, mut keep: impl FnMut(&PlaylistItem) -> bool, @@ -130,9 +124,7 @@ fn retain_playlist_items( let mut groups: IndexMap = IndexMap::new(); let mut outcome = FilterOutcome::default(); for pli in source.into_items() { - outcome.inspected += 1; - if keep(&pli) { - outcome.retained += 1; + if outcome.record(keep(&pli)) { let group_title = pli.header.group.clone(); let cluster = pli.header.xtream_cluster; let cat_id = pli.header.category_id; @@ -148,8 +140,6 @@ fn retain_playlist_items( }) .channels .push(pli); - } else { - outcome.removed += 1; } } @@ -1471,20 +1461,14 @@ impl TransformBuffer { fn apply_filter(&mut self, target: &ConfigTarget) -> FilterOutcome { let mut outcome = FilterOutcome::default(); - self.items.retain(|item| { - outcome.inspected += 1; - let provider = ValueProvider { pli: item, match_as_ascii: false }; - if target.filter(&provider) { - outcome.retained += 1; - true - } else { - outcome.removed += 1; - false - } - }); + self.items.retain(|item| outcome.record(target.filter(&ValueProvider { pli: item, match_as_ascii: false }))); + self.normalize_filter_grouping(); + outcome + } + + fn normalize_filter_grouping(&mut self) { self.grouping = GroupingPolicy::NormalizedCategory; self.reorder_for_grouping(); - outcome } fn apply_rename(&mut self, target: &ConfigTarget) -> Option { @@ -1567,7 +1551,13 @@ fn execute_pipeline_on_items( let mut outcome = PipelineOutcome::default(); for stage in pipe { match stage { - TransformStage::Filter => outcome.filter = Some(buffer.apply_filter(target)), + TransformStage::Filter => { + if target.filter.processing.is_some() { + outcome.filter = Some(buffer.apply_filter(target)); + } else { + buffer.normalize_filter_grouping(); + } + } TransformStage::Rename => outcome.rename = buffer.apply_rename(target), TransformStage::Map => outcome.mapping = buffer.apply_mapping(target, MappingStage::Processing), } @@ -1575,6 +1565,14 @@ fn execute_pipeline_on_items( (buffer.into_groups(), outcome) } +fn apply_persist_filter(target: &ConfigTarget, groups: &mut Vec) { + let Some(filter) = target.filter.persist.as_ref() else { + return; + }; + let outcome = retain_filtered_playlist(groups, filter); + debug!("Target '{}' persist filter outcome: {outcome:?}", target.name); +} + pub(super) fn execute_pipeline_on_groups( groups: Vec, target: &ConfigTarget, @@ -1694,9 +1692,9 @@ async fn prepare_playlist_for_target input_epg_start { - retain_live_items_with_processed_epg(&mut processed_fpl, &new_epg[input_epg_start..]); - } + } + if clear_invalid_epg_ids && processed_fpl.epg.is_some() { + clear_invalid_live_epg_ids(&mut processed_fpl, &new_epg[input_epg_start..]); } if let Some(stat) = stats.get_mut(&processed_fpl.input.name) { stat.processed_stats.group_count = processed_fpl.get_group_count(); @@ -1778,7 +1776,7 @@ async fn finalize_prepared_target (Result<(), Vec>, Vec) { let target = &prepared.target; let mut new_playlist = prepared.playlist; - let new_epg = prepared.epg; + let mut new_epg = prepared.epg; let mut errors = Vec::new(); let broadcast_step = create_broadcast_callback(&ctx.events); let mut step = StepMeasure::new(&target.name, broadcast_step); @@ -1836,6 +1834,9 @@ async fn finalize_prepared_target PlaylistItem { let mut item = PlaylistItem { header: PlaylistItemHeader { @@ -3624,7 +3718,7 @@ match { } #[test] - fn required_epg_removes_ids_invalidated_by_after_epg_mapping() { + fn clear_invalid_epg_ids_clears_ids_invalidated_by_after_epg_mapping() { let runtime = Runtime::new().expect("runtime"); runtime.block_on(async { let dir = tempdir().expect("tempdir"); @@ -3674,7 +3768,7 @@ match { build_mapping("rewrite", MappingStage::AfterEpg, r#"@epg_channel_id = "missing.epg""#); let add_virtual = build_mapping("virtual", MappingStage::AfterEpg, r#"add_favourite("Echo")"#); let mut target = build_target(vec![rewrite_epg, add_virtual], false); - target.options = Some(ConfigTargetOptions { required_epg: true, ..Default::default() }); + target.options = Some(ConfigTargetOptions { clear_invalid_epg_ids: true, ..Default::default() }); let mut stats = HashMap::from([( Arc::clone(&input.name), create_input_stat(1, 1, 0, input.input_type, &input.name, 0), @@ -3693,9 +3787,13 @@ match { .expect("target preparation"); assert!(errors.is_empty()); - assert!(prepared.playlist.is_empty()); - assert_eq!(stats[&input.name].processed_stats.group_count, 0); - assert_eq!(stats[&input.name].processed_stats.channel_count, 0); + assert!(!prepared.playlist.is_empty()); + assert!(prepared + .playlist + .iter() + .flat_map(|group| &group.channels) + .all(|channel| channel.header.epg_channel_id.is_none())); + assert_eq!(stats[&input.name].processed_stats.channel_count, 2); }); } diff --git a/backend/processing/src/processor/xtream.rs b/backend/processing/src/processor/xtream.rs index d69d910e1..e38eb090e 100644 --- a/backend/processing/src/processor/xtream.rs +++ b/backend/processing/src/processor/xtream.rs @@ -1,10 +1,13 @@ +use crate::fetched_playlist::FetchedPlaylist; use log::{debug, warn}; use parking_lot::Mutex; use shared::{ error::TuliproxError, - model::{LiveStreamProperties, StreamProperties, XtreamCluster, XtreamPlaylistItem}, + model::{ + LiveStreamProperties, PlaylistEntry, PlaylistItemType, StreamProperties, XtreamCluster, XtreamPlaylistItem, + }, }; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use tuliprox_core::{ model::{AppConfig, ConfigInput, ConfigInputFlags, ProviderHandle, ProviderIdType}, utils::{ @@ -206,6 +209,55 @@ pub async fn update_live_stream_metadata( Ok(Some(properties)) } +pub(crate) fn sync_resolved_xtream_properties( + provider_fpl: &mut FetchedPlaylist<'_>, + processed_fpl: &mut FetchedPlaylist<'_>, + cluster: XtreamCluster, + item_type: PlaylistItemType, + extract: impl Fn(&StreamProperties) -> Option<&T>, + wrap: impl Fn(Box) -> StreamProperties, +) { + let mut resolved_by_provider_id: HashMap = HashMap::new(); + + for pli in processed_fpl.items() { + if pli.header.xtream_cluster != cluster || pli.header.item_type != item_type { + continue; + } + + let Some(provider_id) = pli.get_provider_id() else { + continue; + }; + if provider_id == 0 { + continue; + } + + if let Some(props) = pli.header.additional_properties.as_ref().and_then(&extract) { + resolved_by_provider_id.entry(provider_id).or_insert_with(|| props.clone()); + } + } + + if resolved_by_provider_id.is_empty() { + return; + } + + for source_pli in provider_fpl.items_mut() { + if source_pli.header.xtream_cluster != cluster || source_pli.header.item_type != item_type { + continue; + } + + let Some(provider_id) = source_pli.get_provider_id() else { + continue; + }; + if provider_id == 0 { + continue; + } + + if let Some(resolved) = resolved_by_provider_id.get(&provider_id) { + source_pli.header.additional_properties = Some(wrap(Box::new(resolved.clone()))); + } + } +} + fn apply_live_probe_success( properties: &mut LiveStreamProperties, raw_video: Option, diff --git a/backend/processing/src/processor/xtream_series.rs b/backend/processing/src/processor/xtream_series.rs index 19ba76fa3..e2f32d53d 100644 --- a/backend/processing/src/processor/xtream_series.rs +++ b/backend/processing/src/processor/xtream_series.rs @@ -22,7 +22,7 @@ use shared::{ }, }; use std::{ - collections::{HashMap, HashSet}, + collections::HashSet, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -148,47 +148,17 @@ async fn playlist_resolve_series_info, processed_fpl: &mut FetchedPlaylist<'_>) { - let mut resolved_series_by_provider_id: HashMap = HashMap::new(); - - for pli in processed_fpl.items() { - if pli.header.xtream_cluster != XtreamCluster::Series || pli.header.item_type != PlaylistItemType::SeriesInfo { - continue; - } - - let Some(provider_id) = pli.get_provider_id() else { - continue; - }; - if provider_id == 0 { - continue; - } - - if let Some(StreamProperties::Series(properties)) = pli.header.additional_properties.as_ref() { - resolved_series_by_provider_id.entry(provider_id).or_insert_with(|| properties.as_ref().clone()); - } - } - - if resolved_series_by_provider_id.is_empty() { - return; - } - - for source_pli in provider_fpl.items_mut() { - if source_pli.header.xtream_cluster != XtreamCluster::Series - || source_pli.header.item_type != PlaylistItemType::SeriesInfo - { - continue; - } - - let Some(provider_id) = source_pli.get_provider_id() else { - continue; - }; - if provider_id == 0 { - continue; - } - - if let Some(resolved) = resolved_series_by_provider_id.get(&provider_id) { - source_pli.header.additional_properties = Some(StreamProperties::Series(Box::new(resolved.clone()))); - } - } + crate::processor::xtream::sync_resolved_xtream_properties( + provider_fpl, + processed_fpl, + XtreamCluster::Series, + PlaylistItemType::SeriesInfo, + |props| match props { + StreamProperties::Series(s) => Some(s.as_ref()), + _ => None, + }, + StreamProperties::Series, + ); } fn queue_background_series_info( diff --git a/backend/processing/src/processor/xtream_vod.rs b/backend/processing/src/processor/xtream_vod.rs index aff1315c7..207c8f19d 100644 --- a/backend/processing/src/processor/xtream_vod.rs +++ b/backend/processing/src/processor/xtream_vod.rs @@ -17,12 +17,12 @@ use shared::{ error::TuliproxError, foundation::ValueProvider, model::{ - EventSink, MediaQuality, PlaylistEntry, PlaylistItem, PlaylistItemType, StreamProperties, - VideoStreamDetailProperties, VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo, + EventSink, MediaQuality, PlaylistItem, PlaylistItemType, StreamProperties, VideoStreamDetailProperties, + VideoStreamProperties, XtreamCluster, XtreamPlaylistItem, XtreamVideoInfo, }, }; use std::{ - collections::{HashMap, HashSet}, + collections::HashSet, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -96,47 +96,17 @@ pub async fn playlist_resolve_vod, processed_fpl: &mut FetchedPlaylist<'_>) { - let mut resolved_vod_by_provider_id: HashMap = HashMap::new(); - - for pli in processed_fpl.items() { - if pli.header.xtream_cluster != XtreamCluster::Video || pli.header.item_type != PlaylistItemType::Video { - continue; - } - - let Some(provider_id) = pli.get_provider_id() else { - continue; - }; - if provider_id == 0 { - continue; - } - - if let Some(StreamProperties::Video(properties)) = pli.header.additional_properties.as_ref() { - resolved_vod_by_provider_id.entry(provider_id).or_insert_with(|| properties.as_ref().clone()); - } - } - - if resolved_vod_by_provider_id.is_empty() { - return; - } - - for source_pli in provider_fpl.items_mut() { - if source_pli.header.xtream_cluster != XtreamCluster::Video - || source_pli.header.item_type != PlaylistItemType::Video - { - continue; - } - - let Some(provider_id) = source_pli.get_provider_id() else { - continue; - }; - if provider_id == 0 { - continue; - } - - if let Some(resolved) = resolved_vod_by_provider_id.get(&provider_id) { - source_pli.header.additional_properties = Some(StreamProperties::Video(Box::new(resolved.clone()))); - } - } + crate::processor::xtream::sync_resolved_xtream_properties( + provider_fpl, + processed_fpl, + XtreamCluster::Video, + PlaylistItemType::Video, + |props| match props { + StreamProperties::Video(v) => Some(v.as_ref()), + _ => None, + }, + StreamProperties::Video, + ); } #[allow(clippy::too_many_lines, clippy::too_many_arguments)] diff --git a/backend/repository/src/alias_repository.rs b/backend/repository/src/alias_repository.rs index a3881e0ab..c702d9758 100644 --- a/backend/repository/src/alias_repository.rs +++ b/backend/repository/src/alias_repository.rs @@ -953,12 +953,11 @@ pub async fn csv_patch_batch_sort_by_exp_date( if aliases.len() < 2 { return Ok(false); } - let mut sorted = aliases.clone(); - sorted.sort_by(|a, b| compare_alias_exp_date_with_order(a, b, order)); - if sorted == aliases { + let compare = |a: &ConfigInputAliasDto, b: &ConfigInputAliasDto| compare_alias_exp_date_with_order(a, b, order); + if aliases.windows(2).all(|pair| compare(&pair[0], &pair[1]) != std::cmp::Ordering::Greater) { return Ok(false); } - aliases = sorted; + aliases.sort_by(compare); csv_write_input_to_path(&file_path, &aliases).map_err(|err| TuliproxError::ConfigInput(format!("{err}"))).await?; Ok(true) } diff --git a/backend/repository/src/epg_repository.rs b/backend/repository/src/epg_repository.rs index b299baa01..2a934acaa 100644 --- a/backend/repository/src/epg_repository.rs +++ b/backend/repository/src/epg_repository.rs @@ -1,4 +1,5 @@ use crate::{ + error_macros::{cant_open_result, cant_query_result}, m3u_get_epg_file_path_for_target, xtream_get_epg_file_path_for_target, xtream_get_storage_path, BPlusTree, BPlusTreeQuery, }; @@ -180,13 +181,12 @@ pub async fn epg_query_channels_by_storage_key( task::spawn_blocking(move || { let _guard = file_lock; let mut query = BPlusTreeQuery::, EpgChannel>::try_new(&epg_path) - .map_err(|e| TuliproxError::RepositoryEpg(format!("failed to open epg db {}: {e}", epg_path.display())))?; + .map_err(|e| cant_open_result!(RepositoryEpg, "epg", &epg_path, e))?; let mut results = Vec::with_capacity(storage_keys.len()); for storage_key in &storage_keys { - let channel = query.query(storage_key).map_err(|e| { - TuliproxError::RepositoryEpg(format!("failed to query epg db {}: {e}", epg_path.display())) - })?; + let channel = + query.query(storage_key).map_err(|e| cant_query_result!(RepositoryEpg, "epg", &epg_path, e))?; results.push((Arc::clone(storage_key), channel)); } Ok(results) @@ -201,7 +201,6 @@ mod tests { use crate::BPlusTree; use arc_swap::ArcSwapOption; use shared::{ - foundation::Filter, model::{ EpgCategory, EpgChannel, EpgProgramme, PlaylistItem, PlaylistItemHeader, ProcessingOrder, XtreamCluster, }, @@ -221,7 +220,7 @@ mod tests { name: "ics-target".to_string(), options: None, sort: None, - filter: Filter::default(), + filter: tuliprox_core::model::StagedFilter::default(), output: vec![ TargetOutput::M3u(M3uTargetOutput { filename: None, diff --git a/backend/repository/src/error_macros.rs b/backend/repository/src/error_macros.rs new file mode 100644 index 000000000..0a674c36f --- /dev/null +++ b/backend/repository/src/error_macros.rs @@ -0,0 +1,90 @@ +//! Shared error-mapping macros for the repository crate. +//! +//! Every backend persists B+Tree-backed playlists through the same shape of +//! call site: read a tree from disk, mutate it, store it back, then bubble +//! `io::Error` / `JoinError` up as `TuliproxError::Repository`. The +//! human-readable message wraps the file path and the underlying error. +//! +//! Historically each submodule (`m3u_repository`, `xtream_repository`) defined +//! its own private `cant_write_result!` macro with the variant baked in, which +//! meant the same boilerplate lived in two places and any new backend had to +//! copy-paste the format string verbatim. Centralising the macros here keeps +//! the format identical across backends and removes a class of +//! "fix-it-in-one-place-but-forget-the-other" drift. +//! +//! All call sites must supply the `TuliproxError` variant explicitly (e.g. +//! `RepositoryM3u`) so the macro cannot accidentally widen an error type. +//! `TuliproxError` is expected to be in scope at every call site (this crate +//! always uses it via `shared::error::TuliproxError`). + +/// Wrap an `io::Error` (or any `Display` value) into the canonical +/// "failed to write {label} playlist: {path} - {err}" message and produce the +/// matching `TuliproxError::Repository`. +/// +/// Use wherever a `BPlusTree::store` / `store_with_index` failure needs to be +/// promoted to a domain error. The `$label` is the short, lower-case +/// identifier for the backend (e.g. `"m3u"`, `"xtream"`). +/// +/// Example: +/// ```ignore +/// tree.store(&path).map_err(|err| { +/// cant_write_result!(RepositoryM3u, "m3u", &path, err) +/// })?; +/// ``` +macro_rules! cant_write_result { + ($variant:ident, $label:literal, $path:expr, $err:expr $(,)?) => {{ + TuliproxError::$variant(format!("failed to write {} playlist: {} - {}", $label, $path.display(), $err)) + }}; +} + +/// Await an async expression and promote a `JoinError` into the matching +/// `TuliproxError::Repository`. The `$fmt` / `$args` mirror +/// `format!`; the join error is appended as the trailing format argument so +/// callers do not need to remember to interpolate it. +/// +/// Example: +/// ```ignore +/// tokio::spawn(work).await.map_err(|err| { +/// await_playlist_write!(RepositoryM3u, expr, "failed to read m3u playlist: {}", path.display()) +/// })??; +/// ``` +macro_rules! await_playlist_write { + ($variant:ident, $expr:expr, $fmt:literal $(, $args:expr)* $(,)?) => {{ + $expr.await.map_err(|err| { + TuliproxError::$variant(format!($fmt $(, $args)*, err)) + })? + }}; +} + +/// Wrap an `io::Error` (or any `Display` value) into the canonical +/// "failed to read {label} playlist: {path} - {err}" message and produce the +/// matching `TuliproxError::Repository`. +macro_rules! cant_read_result { + ($variant:ident, $label:literal, $path:expr, $err:expr $(,)?) => {{ + TuliproxError::$variant(format!("failed to read {} playlist: {} - {}", $label, $path.display(), $err)) + }}; +} + +/// Wrap an `io::Error` (or any `Display` value) into the canonical +/// "failed to open {label} db {path}: {err}" message and produce the +/// matching `TuliproxError::Repository`. +macro_rules! cant_open_result { + ($variant:ident, $label:literal, $path:expr, $err:expr $(,)?) => {{ + TuliproxError::$variant(format!("failed to open {} db {}: {}", $label, $path.display(), $err)) + }}; +} + +/// Wrap an `io::Error` (or any `Display` value) into the canonical +/// "failed to query {label} db {path}: {err}" message and produce the +/// matching `TuliproxError::Repository`. +macro_rules! cant_query_result { + ($variant:ident, $label:literal, $path:expr, $err:expr $(,)?) => {{ + TuliproxError::$variant(format!("failed to query {} db {}: {}", $label, $path.display(), $err)) + }}; +} + +pub(crate) use await_playlist_write; +pub(crate) use cant_open_result; +pub(crate) use cant_query_result; +pub(crate) use cant_read_result; +pub(crate) use cant_write_result; diff --git a/backend/repository/src/lib.rs b/backend/repository/src/lib.rs index e23630e14..3b5a77f34 100644 --- a/backend/repository/src/lib.rs +++ b/backend/repository/src/lib.rs @@ -1,3 +1,4 @@ +mod error_macros; mod storage; mod target_id_mapping; // A B+Tree-backed IPv4 lookup and a CLI dumper for the repository's databases. diff --git a/backend/repository/src/m3u_repository.rs b/backend/repository/src/m3u_repository.rs index fcbf49c3f..eed65702f 100644 --- a/backend/repository/src/m3u_repository.rs +++ b/backend/repository/src/m3u_repository.rs @@ -1,6 +1,7 @@ use super::playlist_mem_cache::PlaylistStorageState; use crate::{ bplustree::{BPlusTree, BPlusTreeQuery}, + error_macros::{await_playlist_write, cant_read_result, cant_write_result}, m3u_playlist_iterator::M3uPlaylistM3uTextIterator, playlist_backend::{ensure_storage_path, iter_raw_playlist, M3u, PlaylistBackend}, playlist_repository::get_input_m3u_playlist_file_path, @@ -32,20 +33,6 @@ use tuliprox_core::{ utils::{async_file_writer, file_exists_async}, }; -macro_rules! cant_write_result { - ($path:expr, $err:expr) => { - TuliproxError::RepositoryM3u(format!("failed to write m3u playlist: {} - {}", $path.display(), $err)) - }; -} - -macro_rules! await_playlist_write { - ($expr:expr, $fmt:literal $(, $args:expr)* ) => {{ - $expr.await.map_err(|err| { - TuliproxError::RepositoryM3u(format!($fmt $(, $args)*, err)) - })? - }}; -} - pub fn m3u_get_file_path_for_db(target_path: &Path) -> PathBuf { target_path.join(storage_const::PATH_M3U).join(concat_string!( storage_const::FILE_M3U, @@ -85,22 +72,38 @@ where F: FnMut(&M3uPlaylistItem) -> Result, { let file = await_playlist_write!( + RepositoryM3u, fs::File::create(m3u_filename), "Can't write m3u plain playlist {} - {}", m3u_filename.display() ); // Larger buffer for sequential writes to reduce syscalls let mut writer = async_file_writer(file); - await_playlist_write!(writer.write_all(b"#EXTM3U\n"), "Failed to write header to {} - {}", m3u_filename.display()); + await_playlist_write!( + RepositoryM3u, + writer.write_all(b"#EXTM3U\n"), + "Failed to write header to {} - {}", + m3u_filename.display() + ); for m3u in m3u_playlist { let line = build_line(m3u)?; let bytes = line.as_bytes(); - await_playlist_write!(writer.write_all(bytes), "Failed to write entry to {} - {}", m3u_filename.display()); - await_playlist_write!(writer.write_all(b"\n"), "Failed to write newline to {} - {}", m3u_filename.display()); + await_playlist_write!( + RepositoryM3u, + writer.write_all(bytes), + "Failed to write entry to {} - {}", + m3u_filename.display() + ); + await_playlist_write!( + RepositoryM3u, + writer.write_all(b"\n"), + "Failed to write newline to {} - {}", + m3u_filename.display() + ); } - await_playlist_write!(writer.flush(), "Failed to flush {} - {}", m3u_filename.display()); + await_playlist_write!(RepositoryM3u, writer.flush(), "Failed to flush {} - {}", m3u_filename.display()); Ok(()) } @@ -120,7 +123,12 @@ where { let tmp_path = temp_m3u_filename(m3u_filename); write_m3u_text_file(&tmp_path, m3u_playlist, build_line).await?; - await_playlist_write!(fs::rename(&tmp_path, m3u_filename), "Failed to replace {} - {}", m3u_filename.display()); + await_playlist_write!( + RepositoryM3u, + fs::rename(&tmp_path, m3u_filename), + "Failed to replace {} - {}", + m3u_filename.display() + ); Ok(()) } @@ -217,6 +225,7 @@ async fn persist_m3u_playlist_as_text( if let Err(rename_err) = async { await_playlist_write!( + RepositoryM3u, fs::rename(&provider_tmp, &provider_filename), "Failed to replace {} - {}", provider_filename.display() @@ -232,6 +241,7 @@ async fn persist_m3u_playlist_as_text( if let Err(rename_err) = async { await_playlist_write!( + RepositoryM3u, fs::rename(&m3u_tmp, &m3u_filename), "Failed to replace {} - {}", m3u_filename.display() @@ -286,7 +296,7 @@ pub async fn m3u_write_playlist( tree.insert(m3u.virtual_id, m3u); } tree.store_with_index(&m3u_path_clone, |pli| pli.source_ordinal) - .map_err(|err| cant_write_result!(&m3u_path_clone, err))?; + .map_err(|err| cant_write_result!(RepositoryM3u, "m3u", &m3u_path_clone, err))?; Ok(()) }) .await @@ -399,7 +409,7 @@ pub async fn persist_input_m3u_playlist( for m3u in &playlist_items { tree.insert(m3u.provider_id.clone(), m3u.clone()); } - tree.store(&m3u_path_clone).map_err(|err| cant_write_result!(&m3u_path_clone, err))?; + tree.store(&m3u_path_clone).map_err(|err| cant_write_result!(RepositoryM3u, "m3u", &m3u_path_clone, err))?; Ok(()) }) .await @@ -485,9 +495,7 @@ pub async fn load_input_m3u_playlist( Ok(groups.into_values().collect()) }) .await - .map_err(|err| { - TuliproxError::RepositoryM3u(format!("failed to read m3u playlist: {} - {err}", m3u_path_err.display())) - })??; + .map_err(|err| cant_read_result!(RepositoryM3u, "m3u", &m3u_path_err, err))??; Ok(groups) } diff --git a/backend/repository/src/playlist_repository.rs b/backend/repository/src/playlist_repository.rs index 2d817d64e..6b86915c6 100644 --- a/backend/repository/src/playlist_repository.rs +++ b/backend/repository/src/playlist_repository.rs @@ -142,7 +142,7 @@ pub async fn persist_playlist( for output in &target.output { let mut filtered: Option> = - output.filter().and_then(|flt| apply_filter_to_playlist(playlist, flt)); + output.filter().map(|flt| apply_filter_to_playlist(playlist, flt)); let pl: &mut [PlaylistGroup] = if let Some(filtered_playlist) = filtered.as_mut() { filtered_playlist.as_mut_slice() } else { playlist }; diff --git a/backend/repository/src/xtream_repository.rs b/backend/repository/src/xtream_repository.rs index 6d1ff81f2..4705e6436 100644 --- a/backend/repository/src/xtream_repository.rs +++ b/backend/repository/src/xtream_repository.rs @@ -4,6 +4,7 @@ use crate::{ ensure_distinct_sidecar_lock_domains, publish_staged_database, BPlusTree, BPlusTreeError, BPlusTreeQuery, BPlusTreeStagingArtifacts, BPlusTreeUpdate, FlushPolicy, }, + error_macros::{cant_read_result, cant_write_result}, playlist_backend::{ensure_storage_path, iter_raw_playlist, PlaylistBackend, PlaylistKey, Xtream}, playlist_scratch::PlaylistScratch, storage::{ @@ -50,12 +51,6 @@ use tuliprox_core::{ use tuliprox_parser::xtream; use uuid::Uuid; -macro_rules! cant_write_result { - ($path:expr, $err:expr) => { - TuliproxError::RepositoryXtream(format!("failed to write xtream playlist: {} - {}", $path.display(), $err)) - }; -} - #[inline] pub fn get_collection_path(path: &Path, collection: &str) -> PathBuf { path.join(format!("{collection}.json")) } @@ -135,7 +130,7 @@ where }) .await .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(&xtream_path, err))?; + .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; } Ok(()) } @@ -175,7 +170,7 @@ pub async fn write_playlist_item_update( }) .await .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(&xtream_path, err))?; + .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; Ok(()) } @@ -221,7 +216,7 @@ pub async fn write_playlist_batch_item_upsert( }) .await .map_err(|e| TuliproxError::RepositoryXtream(format!("Blocking task failed: {e}")))? - .map_err(|err| cant_write_result!(&xtream_path, err))?; + .map_err(|err| cant_write_result!(RepositoryXtream, "xtream", &xtream_path, err))?; Ok(()) } @@ -1882,8 +1877,7 @@ pub async fn load_input_xtream_playlist( // Load Items let file_lock = app_config.file_locks.read_lock(&xtream_path).await; - let xtream_display = xtream_path.display().to_string(); - let xtream_path = xtream_path.clone(); + let xtream_path_err = xtream_path.clone(); let items = tokio::task::spawn_blocking(move || -> Result, TuliproxError> { let _guard = file_lock; let mut items = Vec::new(); @@ -1896,9 +1890,7 @@ pub async fn load_input_xtream_playlist( Ok(items) }) .await - .map_err(|err| { - TuliproxError::RepositoryXtream(format!("failed to read xtream playlist: {xtream_display} - {err}")) - })??; + .map_err(|err| cant_read_result!(RepositoryXtream, "xtream", &xtream_path_err, err))??; for item in items { let cat_id = item.category_id; diff --git a/config/source.yml b/config/source.yml index 4001c5f27..83829ec7a 100644 --- a/config/source.yml +++ b/config/source.yml @@ -26,7 +26,7 @@ sources: filter: "!final_channel_lineup!" options: ignore_logo: false - required_epg: false + clear_invalid_epg_ids: false epg_output: lowercase_ids: false lowercase_xmltv_display_names: false @@ -66,7 +66,7 @@ sources: filter: "!final_channel_lineup!" options: ignore_logo: false - required_epg: false + clear_invalid_epg_ids: false epg_output: lowercase_ids: false lowercase_xmltv_display_names: false diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index 67b7df382..1743cfeaf 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -1108,7 +1108,9 @@ sources: - my_provider targets: - name: my_target - filter: 'Group ~ ".*"' + filter: + processing: 'Group ~ "News"' + persist: 'EpgId IS NOT EMPTY' output: - type: m3u ``` @@ -1145,7 +1147,7 @@ sources: sort: { } options: ignore_logo: false - required_epg: false + clear_invalid_epg_ids: false epg_output: lowercase_ids: false lowercase_xmltv_display_names: false @@ -1175,20 +1177,20 @@ sources: #### Target Parameters -| Parameter | Type | Required | Default | Technical Impact & Background | -|:-------------------|:-------|:--------:|:----------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `enabled` | Bool | No | `true` | If set to `false`, Tuliprox skips building this target during normal processing. This reduces CPU, disk, and upstream workload, but the target can still be selected explicitly via CLI target execution if matched by `-t`. | -| `name` | String | No | `default` | Logical target name. If not `default`, it must be unique. Unique names are important for selective execution (`-t `) and for clearly separating output identities in Tuliprox's processing pipeline. | -| `processing_order` | Enum | No | `frm` | Defines execution order for **F**ilter, **R**ename, and **M**ap. This directly changes which intermediate state downstream steps operate on and can therefore materially alter the final playlist result. | -| `filter` | String | Yes | | Global filter DSL expression for the target. This determines which entries survive into the final target after the selected processing order has been applied. | -| `rename` | List | No | | Regex-based transformations applied to selected fields. This is commonly used to normalize channel/group labels before sorting, mapping, or export. | -| `mapping` | List | No | | References mapping IDs from `mapping.yml` for advanced transformation logic. This is where deep structural rewriting and metadata normalization can be applied. | -| `sort` | Object | No | | Defines ordering for groups and channels after transformations. This affects the final playlist structure seen by clients and can significantly improve navigation quality in IPTV players. | -| `options` | Object | No | | Target-level behavior switches such as logo suppression, duplicate removal, and shared live-stream handling. These options influence memory usage, playlist cleanliness, and reverse-proxy behavior. | -| `output` | List | Yes | | Mandatory list of output formats. A single target can generate multiple output representations (e.g., `xtream`, `m3u`, `strm`, `hdhomerun`) from the same transformed result set. | -| `favourites` | List | No | | Duplicates final transformed channels into dedicated favorite groups after processing is complete. This adds curated views without changing the original group structure. | -| `watch` | List | No | | Defines watched group patterns. If matching groups change during updates, Tuliprox emits Messaging events so operational changes become observable automatically. | -| `use_memory_cache` | Bool | No | `false` | If enabled, the final compiled playlist is cached in RAM. This reduces disk access and improves delivery speed, especially for M3U downloads, but increases memory consumption. | +| Parameter | Type | Required | Default | Technical Impact & Background | +|:-------------------|:--------------|:--------:|:----------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `enabled` | Bool | No | `true` | If set to `false`, Tuliprox skips building this target during normal processing. This reduces CPU, disk, and upstream workload, but the target can still be selected explicitly via CLI target execution if matched by `-t`. | +| `name` | String | No | `default` | Logical target name. If not `default`, it must be unique. Unique names are important for selective execution (`-t `) and for clearly separating output identities in Tuliprox's processing pipeline. | +| `processing_order` | Enum | No | `frm` | Defines execution order for **F**ilter, **R**ename, and **M**ap. This directly changes which intermediate state downstream steps operate on and can therefore materially alter the final playlist result. | +| `filter` | String or Map | No | | Optional target filter. A string is the backward-compatible `processing` filter. A map can define optional `processing` and `persist` stages. | +| `rename` | List | No | | Regex-based transformations applied to selected fields. This is commonly used to normalize channel/group labels before sorting, mapping, or export. | +| `mapping` | List | No | | References mapping IDs from `mapping.yml` for advanced transformation logic. This is where deep structural rewriting and metadata normalization can be applied. | +| `sort` | Object | No | | Defines ordering for groups and channels after transformations. This affects the final playlist structure seen by clients and can significantly improve navigation quality in IPTV players. | +| `options` | Object | No | | Target-level behavior switches such as logo suppression, duplicate removal, and shared live-stream handling. These options influence memory usage, playlist cleanliness, and reverse-proxy behavior. | +| `output` | List | Yes | | Mandatory list of output formats. A single target can generate multiple output representations (e.g., `xtream`, `m3u`, `strm`, `hdhomerun`) from the same transformed result set. | +| `favourites` | List | No | | Duplicates final transformed channels into dedicated favorite groups after processing is complete. This adds curated views without changing the original group structure. | +| `watch` | List | No | | Defines watched group patterns. If matching groups change during updates, Tuliprox emits Messaging events so operational changes become observable automatically. | +| `use_memory_cache` | Bool | No | `false` | If enabled, the final compiled playlist is cached in RAM. This reduces disk access and improves delivery speed, especially for M3U downloads, but increases memory consumption. | --- @@ -1220,8 +1222,24 @@ opt into `stage: after_epg` always run once EPG enrichment has completed, regard ### 3.2.2 `filter` -The target-level `filter` is a string-based expression using Tuliprox's filter DSL. -It defines which entries remain in the final target after the selected processing stages have been applied. +The target-level `filter` uses Tuliprox's filter DSL and is optional. The scalar form remains backward compatible and +runs at the `F` position of `processing_order`: + +```yaml +filter: 'Group ~ "Sports.*"' +``` + +To filter the final transformed state, use the staged form. Both fields are optional, but at least one must be present: + +```yaml +filter: + processing: 'Type = live' + persist: 'EpgId IS NOT EMPTY' +``` + +`processing` runs at the normal `F` position. `persist` runs after EPG matching, smart matching, all mappings, merge, +favourites/Trakt, deduplication, sorting, channel numbering, and counters, immediately before watch evaluation and target +persistence. Output-level filters remain plain strings and have no configurable stage. You can define complex strings or regex patterns exactly once in [template.yml](./template.md) and call them by wrapping the template name in exclamation marks: `!MACRO_NAME!`. @@ -1235,11 +1253,15 @@ Tuliprox supports the following filter expression types: * Regular expression comparison: `([fieldname]) ~ "regexp"`
The `[fieldname]` can be `Group`, `Title`, `Name`, `Caption`, `Url`, `Genre`, `Input`, `EpgId` or `Type`. * String comparison (case-insensitive, no regex needed): - * Exact: `Group = "Sports"` / negated: `Group != "Sports"` + * Equal (`=`): `Group = "Sports"` matches the complete text `Sports`. + * Not equal (`!=`): `Group != "Sports"` matches every other group name. * Substring: `Title CONTAINS "HD"` * Prefix: `Caption STARTSWITH "DE:"` * Case-insensitivity is ASCII-only: ASCII letters match regardless of case, non-ASCII characters must match exactly. `Title CONTAINS "cinéma"` matches `Cinéma` but not `CINÉMA`. +* Presence comparison: `EpgId IS EMPTY` matches a missing or empty field; `EpgId IS NOT EMPTY` matches a populated field. + This is especially useful in a `persist` filter after `clear_invalid_epg_ids` has removed unresolved EPG IDs. + `EpgId = EMPTY` and `EpgId != EMPTY` are accepted as aliases and normalize to the `IS` forms. * Set membership (case-insensitive exact match against a list): `Group IN ["Sports", "News"]` * Numeric comparison on the channel number: `Chno = 5`, `Chno != 5`, `Chno > 100`, `Chno >= 100`, `Chno < 200`, `Chno <= 200` * Numeric comparison on the detected quality tier: `Quality >= 3`
@@ -1274,6 +1296,16 @@ This example keeps: * entries from groups starting with `DE`, except titles containing `Shopping` * all entries from groups starting with `AU` +#### Understanding filters without technical background + +Think of a filter as a set of questions that Tuliprox asks about every channel. `Group = "Sports"` asks whether the +complete group name is `Sports`, while `Group != "Sports"` asks whether it is anything else. `CONTAINS` searches for a +piece of text, `STARTSWITH` checks the beginning, and `IS EMPTY` checks whether a value is missing. Join questions with +`AND` when all of them must be true, with `OR` when one is enough, and put `NOT` before a question to reverse it. +Parentheses make clear which questions belong together. Text values always use quotes: `Group = "EMPTY"` searches for +the literal group name `EMPTY`, whereas `EpgId = EMPTY` without quotes is the short form of `EpgId IS EMPTY` and checks +for a missing EPG ID. In practice, start with one simple question and add further conditions only when needed. + --- ### 3.2.3 `rename` @@ -1439,7 +1471,7 @@ targets: use_output: xtream options: ignore_logo: false - required_epg: false + clear_invalid_epg_ids: false epg_output: lowercase_ids: true lowercase_xmltv_display_names: false @@ -1458,7 +1490,7 @@ targets: | Parameter | Type | Required | Default | Technical Impact & Background | |:-------------------------------------------|:-----|:--------:|:--------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `ignore_logo` | Bool | No | `false` | Ignores `tvg-logo` and `tvg-logo-small` attributes. This reduces downstream device-side logo caching and can keep generated M3U playlists leaner for clients with limited storage or poor cache invalidation behavior. | -| `required_epg` | Bool | No | `false` | Keeps only live playlist entries whose EPG ID resolves to programme data from an available EPG source. Filtering and mappings in the regular processing stage run first, so EPG matching only processes the reduced target playlist. VOD, series, and local-library entries are unaffected. If no EPG source was successfully materialized for the input, this option leaves its live entries unchanged. | +| `clear_invalid_epg_ids` | Bool | No | `false` | Clears an EPG ID when it does not resolve to the processed EPG data. Playlist entries are never removed. Smart matching runs first, and IDs introduced or changed by later mappings are validated again before the `persist` filter. The legacy input name `required_epg` is still accepted, but configuration is serialized with the new name. | | `share_live_streams.hls` | Bool | No | `false` | Enables HLS live sharing for the new HLS cache proxy path. This is a configuration switch for the HLS cache feature and is independent from MPEG-TS stream sharing. | | `share_live_streams.mpeg_ts` | Bool | No | `false` | Allows Tuliprox to share MPEG-TS live stream connections in reverse proxy mode. This can reduce upstream provider connection usage when multiple clients watch the same channel, but it increases memory usage per shared channel. | | `remove_duplicates` | Bool | No | `false` | Legacy pre-transform identity deduplication. It runs independently for each input before the F/R/M pipe and removes repeated source identities before mapping can emit additional items. The field remains supported for backward compatibility. | diff --git a/docs/src/examples-recipes.md b/docs/src/examples-recipes.md index 8dd24a201..7d954edd0 100644 --- a/docs/src/examples-recipes.md +++ b/docs/src/examples-recipes.md @@ -32,10 +32,6 @@ update_on_boot: true **2. `source.yml` (Inputs & Targets):** ```yaml -templates: - - name: ALL_CHAN - value: 'Group ~ ".*"' - inputs: - type: xtream name: my_provider @@ -50,10 +46,10 @@ sources: - name: clean_list output: - type: xtream - filter: "!ALL_CHAN!" # Lets everything through ``` -*Here we define the input source based on the provider's information and create a 1:1 mapped target (`clean_list`).* +*Here we define the input source based on the provider's information and create a 1:1 mapped target (`clean_list`). +The optional target filter is omitted, so every entry passes the processing stage.* **3. `api-proxy.yml` (Servers & Users):** diff --git a/frontend/public/assets/i18n/ar.json b/frontend/public/assets/i18n/ar.json index 6d42def5f..929ea7853 100644 --- a/frontend/public/assets/i18n/ar.json +++ b/frontend/public/assets/i18n/ar.json @@ -379,7 +379,7 @@ "WATCH": "قائمة أنماط المجموعات لمراقبة التغييرات. يتم إرسال الإشعارات عبر المراسلة المكوّنة." }, "CONFIG_TARGET_OPTIONS": { - "REQUIRED_EPG": "يحتفظ فقط بالقنوات المباشرة التي تتوفر لها بيانات EPG مطابقة. لا تتأثر عناصر الفيديو حسب الطلب والمسلسلات والمكتبة المحلية.", + "CLEAR_INVALID_EPG_IDS": "يمسح معرفات EPG التي لا تطابق بيانات EPG المعالجة دون إزالة عناصر قائمة التشغيل.", "SHARE_LIVE_STREAMS": "يُفعّل مشاركة البث المباشر. المفتاح الرئيسي يضبط HLS و MPEG-TS معًا؛ يمكن تغيير كل تنسيق أيضًا بشكل منفصل." }, "CONFIG_TARGET_SHARE_LIVE_STREAMS": { @@ -768,6 +768,7 @@ "USERNAME": "المستخدم الذي سيتم تضمين بياناته في عناوين URL لـ .strm." }, "TARGET_FORM": { + "PERSIST_FILTER": "مرشح نهائي اختياري يُطبّق بعد معالجة EPG وجميع التعيينات، مباشرة قبل الحفظ.", "FILTER": "تعبير تصفية عام مطبق على الهدف قبل إنشاء مخرجات صريحة.", "FORCE_REDIRECT": "يُجبر البث مباشرة على الوكيل أو المزود بدلاً من الحل بشكل فردي.", "PROCESSING_ORDER": "يحدد الأولوية أو التسلسل الذي يتم به مطابقة قنوات الهدف وإدراجها." @@ -1325,6 +1326,7 @@ "PROBE_FILTER": "مرشح الفحص", "PROBE_STREAM": "فحص البث", "PROCESSING_ORDER": "ترتيب المعالجة", + "PERSIST_FILTER": "مرشح الحفظ", "PROCESS_PARALLEL": "معالجة متوازية", "PROPERTIES": "خصائص", "PROPRIETARY_DISCOVERY": "اكتشاف احتكاري", @@ -1370,7 +1372,7 @@ "REGEXP": "تعبير منتظم", "RELEASES": "الإصدارات", "REMOVE_DUPLICATES": "إزالة التكرارات", - "REQUIRED_EPG": "اشتراط EPG للقنوات المباشرة", + "CLEAR_INVALID_EPG_IDS": "مسح معرفات EPG غير الصالحة", "RENAME": "إعادة تسمية", "RENAME_SETTINGS": "إعادة تسمية الإعدادات", "REPEAT_PASSWORD": "تكرار كلمة المرور", diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 506f0f34b..0635504a7 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -379,7 +379,7 @@ "WATCH": "List of group patterns to monitor for changes. Notifications are sent via configured messaging." }, "CONFIG_TARGET_OPTIONS": { - "REQUIRED_EPG": "Keeps only live channels that match programme data from an available EPG source. VOD, series, and local library items are unaffected.", + "CLEAR_INVALID_EPG_IDS": "Clears EPG IDs that do not match the processed EPG data. Playlist entries are not removed.", "SHARE_LIVE_STREAMS": "Enables live stream sharing. The master toggle sets HLS and MPEG-TS together; each format can also be changed separately." }, "EPG_OUTPUT_OPTIONS": { @@ -772,6 +772,7 @@ "USERNAME": "The user whose credentials will be embedded in the .strm URLs." }, "TARGET_FORM": { + "PERSIST_FILTER": "Optional final filter applied after EPG processing and all mappings, immediately before persistence.", "FILTER": "Global filter expression applied to the target before creating explicit outputs.", "FORCE_REDIRECT": "Forces streams directly to the proxy or the provider instead of resolving individually.", "PROCESSING_ORDER": "Determines the priority or the sequence in which target channels are matched and listed." @@ -1338,6 +1339,7 @@ "PROBE_FILTER": "Probe Filter", "PROBE_STREAM": "Probe Stream", "PROCESSING_ORDER": "Processing order", + "PERSIST_FILTER": "Persist filter", "PROCESS_PARALLEL": "Process Parallel", "PROPERTIES": "Properties", "PROPRIETARY_DISCOVERY": "Proprietary Discovery", @@ -1455,7 +1457,7 @@ "REGEXP": "Regexp", "RELEASES": "Releases", "REMOVE_DUPLICATES": "Remove duplicates", - "REQUIRED_EPG": "Require EPG for live channels", + "CLEAR_INVALID_EPG_IDS": "Clear invalid EPG IDs", "RENAME": "Rename", "RENAME_SETTINGS": "Rename Settings", "REPEAT_PASSWORD": "Repeat Password", diff --git a/frontend/public/assets/i18n/ru.json b/frontend/public/assets/i18n/ru.json index 9c5599dde..2d8534eb4 100644 --- a/frontend/public/assets/i18n/ru.json +++ b/frontend/public/assets/i18n/ru.json @@ -356,7 +356,7 @@ "WATCH": "Список шаблонов групп для мониторинга изменений. Уведомления отправляются через настроенный мессенджер." }, "CONFIG_TARGET_OPTIONS": { - "REQUIRED_EPG": "Оставляет только прямые каналы, для которых найдены данные программ, но только при наличии успешно загруженного источника EPG. Если такой источник отсутствует, плейлист не изменяется. VOD, сериалы и элементы локальной библиотеки не затрагиваются." + "CLEAR_INVALID_EPG_IDS": "Очищает идентификаторы EPG, не совпадающие с обработанными данными EPG, не удаляя элементы плейлиста." }, "EPG_OUTPUT_OPTIONS": { "LOWERCASE_IDS": "Преобразует технические идентификаторы EPG в нижний регистр ASCII во всех выходах M3U, Xtream, XMLTV и API EPG. После изменения этой опции требуется полное обновление целевого плейлиста.", @@ -731,6 +731,7 @@ "USERNAME": "Пользователь, чьи учетные данные будут встроены в URL-адреса .strm." }, "TARGET_FORM": { + "PERSIST_FILTER": "Необязательный финальный фильтр после обработки EPG и всех сопоставлений, непосредственно перед сохранением.", "FILTER": "Глобальное выражение фильтра, применяемое к цели перед созданием явных выводов.", "FORCE_REDIRECT": "Принудительно направляет потоки напрямую на прокси или провайдера, вместо разрешения по отдельности.", "PROCESSING_ORDER": "Определяет приоритет или последовательность, в которой каналы цели сопоставляются и перечисляются." @@ -1266,6 +1267,7 @@ "PROBE_FILTER": "Фильтр зондирования", "PROBE_STREAM": "Зондировать поток", "PROCESSING_ORDER": "Порядок обработки", + "PERSIST_FILTER": "Фильтр сохранения", "PROCESS_PARALLEL": "Параллельная обработка", "PROPERTIES": "Свойства", "PROPRIETARY_DISCOVERY": "Проприетарное обнаружение", @@ -1311,7 +1313,7 @@ "REGEXP": "Регулярное выражение", "RELEASES": "Релизы", "REMOVE_DUPLICATES": "Удалить дубликаты", - "REQUIRED_EPG": "Требовать EPG для прямых каналов", + "CLEAR_INVALID_EPG_IDS": "Очистить недействительные EPG ID", "RENAME": "Переименовать", "RENAME_SETTINGS": "Настройки переименования", "REPEAT_PASSWORD": "Повторите пароль", diff --git a/frontend/src/app/components/config/api_config_view.rs b/frontend/src/app/components/config/api_config_view.rs index ef84a7708..eff60e035 100644 --- a/frontend/src/app/components/config/api_config_view.rs +++ b/frontend/src/app/components/config/api_config_view.rs @@ -20,10 +20,9 @@ use crate::{ }; use shared::{ concat_string, - error::TuliproxError, model::{ApiProxyConfigDto, ApiProxyServerInfoDto, ConfigApiDto, SortOrder}, }; -use std::{fmt::Display, rc::Rc, str::FromStr}; +use std::{rc::Rc, str::FromStr}; use yew::prelude::*; const LABEL_NAME: &str = "LABEL.NAME"; @@ -43,39 +42,12 @@ const MSG_NON_UNIQUE_SERVER_NAME: &str = "MESSAGES.SAVE.API_PROXY_CONFIG.NON_UNI const SERVER_HEADERS: [&str; 8] = ["EMPTY", "NAME", "PROTOCOL", "HOST", "PORT", "TIMEZONE", "MESSAGE", "PATH"]; -#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, strum_macros::Display, strum_macros::EnumString)] enum ServerTableAction { Delete, Edit, } -impl Display for ServerTableAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Delete => "Delete", - Self::Edit => "Edit", - } - ) - } -} - -impl FromStr for ServerTableAction { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - if s.eq("Delete") { - Ok(Self::Delete) - } else if s.eq("Edit") { - Ok(Self::Edit) - } else { - Err(TuliproxError::Config(format!("Unknown Server Action: {s}"))) - } - } -} - #[derive(Debug, Clone, Copy, Eq, PartialEq)] enum ServerDialogMode { Add, diff --git a/frontend/src/app/components/config/macros.rs b/frontend/src/app/components/config/macros.rs index 7b8feea74..824c1c41e 100644 --- a/frontend/src/app/components/config/macros.rs +++ b/frontend/src/app/components/config/macros.rs @@ -199,6 +199,25 @@ macro_rules! config_field_child { }; } +#[macro_export] +macro_rules! config_field_tags { + ($config:expr, $label:expr, $field:ident, $field_id:expr) => { + $crate::config_field_child!($label, $field_id, { + html! { +
+ { + if $config.$field.is_empty() { + html! {} + } else { + html! { for t in $config.$field.iter() { <$crate::app::components::Chip label={t.clone()} /> } } + } + } +
+ } + }) + }; +} + #[macro_export] macro_rules! config_field_empty { ($label:expr, $field_id:expr) => { diff --git a/frontend/src/app/components/config/messaging_config_view.rs b/frontend/src/app/components/config/messaging_config_view.rs index c83066274..3b72e4b0c 100644 --- a/frontend/src/app/components/config/messaging_config_view.rs +++ b/frontend/src/app/components/config/messaging_config_view.rs @@ -11,7 +11,7 @@ use crate::{ ConfigContext, }, config_field, config_field_bool, config_field_bool_empty, config_field_child, config_field_custom, - config_field_empty, config_field_hide, config_field_optional, edit_field_bool, edit_field_list, + config_field_empty, config_field_hide, config_field_optional, config_field_tags, edit_field_bool, edit_field_list, edit_field_number_f64, edit_field_number_u64, edit_field_text, edit_field_text_option, generate_form_reducer, i18n::use_translation, }; @@ -262,19 +262,7 @@ pub fn MessagingConfigView() -> Html {

{translate.t("LABEL.TELEGRAM")}

{ config_field_hide!(entry, translate.t(LABEL_BOT_TOKEN), bot_token) } - { config_field_child!(translate.t(LABEL_CHAT_IDS), "MESSAGING_CONFIG.TELEGRAM_CHAT_IDS", { - html! { -
- { - if entry.chat_ids.is_empty() { - html! {} - } else { - html! { for t in entry.chat_ids.iter() { } } - } - } -
- } - })} + { config_field_tags!(entry, translate.t(LABEL_CHAT_IDS), chat_ids, "MESSAGING_CONFIG.TELEGRAM_CHAT_IDS") } { config_field_bool!(entry, translate.t(LABEL_MARKDOWN), markdown) } { render_templates_view(&entry.templates) } @@ -295,19 +283,7 @@ pub fn MessagingConfigView() -> Html {

{translate.t(LABEL_REST)}

{ config_field!(entry, translate.t(LABEL_URL), url) } { config_field_optional!(entry, translate.t(LABEL_METHOD), method) } - { config_field_child!(translate.t(LABEL_HEADERS), "MESSAGING_CONFIG.REST_HEADERS", { - html! { -
- { - if entry.headers.is_empty() { - html! {} - } else { - html! { for h in entry.headers.iter() { } } - } - } -
- } - })} + { config_field_tags!(entry, translate.t(LABEL_HEADERS), headers, "MESSAGING_CONFIG.REST_HEADERS") } { render_templates_view(&entry.templates) } }, diff --git a/frontend/src/app/components/config/plans_view.rs b/frontend/src/app/components/config/plans_view.rs index 7f1aff04c..ad75fc574 100644 --- a/frontend/src/app/components/config/plans_view.rs +++ b/frontend/src/app/components/config/plans_view.rs @@ -11,10 +11,9 @@ use crate::{ }; use shared::{ concat_string, - error::TuliproxError, model::{ClusterFlags, PlansConfigDto, ProxyType, SortOrder, UserPlanDto, UserPlanTrialDto}, }; -use std::{fmt::Display, rc::Rc, str::FromStr}; +use std::{rc::Rc, str::FromStr}; use web_sys::MouseEvent; use yew::{platform::spawn_local, prelude::*}; @@ -28,32 +27,11 @@ enum PlanDialogMode { Edit(usize), } +#[derive(Debug, Clone, Copy, Eq, PartialEq, strum_macros::Display, strum_macros::EnumString)] enum PlanTableAction { Delete, Edit, } -impl Display for PlanTableAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Delete => "Delete", - Self::Edit => "Edit", - } - ) - } -} -impl FromStr for PlanTableAction { - type Err = TuliproxError; - fn from_str(s: &str) -> Result { - match s { - "Delete" => Ok(Self::Delete), - "Edit" => Ok(Self::Edit), - _ => Err(TuliproxError::Config(format!("Unknown Plan Action: {s}"))), - } - } -} fn build_default_plan(existing_plans: &[UserPlanDto]) -> UserPlanDto { let mut index = existing_plans.len() + 1; diff --git a/frontend/src/app/components/dashboard/stream_display/mod.rs b/frontend/src/app/components/dashboard/stream_display/mod.rs index a5d18e552..77513c49f 100644 --- a/frontend/src/app/components/dashboard/stream_display/mod.rs +++ b/frontend/src/app/components/dashboard/stream_display/mod.rs @@ -23,20 +23,14 @@ use gloo_timers::callback::Interval; pub use helpers::get_stream_info_config; use shared::{ defaults::default_kick_secs, - error::TuliproxError, model::{ PlaylistItemType, PlaylistRequest, PlaylistUrlResolveRequest, ProtocolMessage, StreamInfo, StreamInfoConfigDto, UserCommand, VirtualId, }, }; -use std::{collections::HashMap, fmt::Display, rc::Rc, str::FromStr}; +use std::{collections::HashMap, rc::Rc, str::FromStr}; use yew::{platform::spawn_local, prelude::*}; -const KICK: &str = "kick"; -const COPY_LINK_TULIPROX_VIRTUAL_ID: &str = "copy_link_tuliprox_virtual_id"; -const COPY_LINK_TULIPROX_WEBPLAYER_URL: &str = "copy_link_tuliprox_webplayer_url"; -const COPY_LINK_PROVIDER_URL: &str = "copy_link_provider_url"; - fn stream_display_key(stream: &StreamInfo) -> String { // Prefer a stable session identity so archive HLS segment addr/uid churn does not remount the row. if let Some(token) = stream.session_token.as_deref().filter(|token| !token.is_empty()) { @@ -421,39 +415,11 @@ mod tests { } } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, strum_macros::Display, strum_macros::EnumString)] +#[strum(serialize_all = "snake_case")] enum StreamDisplayAction { Kick, CopyLinkTuliproxVirtualId, CopyLinkTuliproxWebPlayerUrl, CopyLinkProviderUrl, } - -impl Display for StreamDisplayAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Kick => KICK, - Self::CopyLinkTuliproxVirtualId => COPY_LINK_TULIPROX_VIRTUAL_ID, - Self::CopyLinkTuliproxWebPlayerUrl => COPY_LINK_TULIPROX_WEBPLAYER_URL, - Self::CopyLinkProviderUrl => COPY_LINK_PROVIDER_URL, - } - ) - } -} - -impl FromStr for StreamDisplayAction { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s { - KICK => Ok(Self::Kick), - COPY_LINK_TULIPROX_VIRTUAL_ID => Ok(Self::CopyLinkTuliproxVirtualId), - COPY_LINK_TULIPROX_WEBPLAYER_URL => Ok(Self::CopyLinkTuliproxWebPlayerUrl), - COPY_LINK_PROVIDER_URL => Ok(Self::CopyLinkProviderUrl), - _ => Err(TuliproxError::Config(format!("Unknown Stream Action: {s}"))), - } - } -} diff --git a/frontend/src/app/components/playlist/filter_view.rs b/frontend/src/app/components/playlist/filter_view.rs index 060655b1d..ee60e560c 100644 --- a/frontend/src/app/components/playlist/filter_view.rs +++ b/frontend/src/app/components/playlist/filter_view.rs @@ -100,6 +100,15 @@ fn render_filter(filter: &Filter, pretty: bool, level: usize, do_indent: bool, p }, + Filter::PresenceComparison(field, op) => html! { + <> + { indent(level, do_indent && pretty) } + + {format!("{}", field)} + {format!(" {}", op)} + + + }, Filter::NumericComparison(field, op, value) => html! { <> { indent(level, do_indent && pretty) } diff --git a/frontend/src/app/components/playlist/mapper_script_view.rs b/frontend/src/app/components/playlist/mapper_script_view.rs index 6f8a16038..a9ba9dffa 100644 --- a/frontend/src/app/components/playlist/mapper_script_view.rs +++ b/frontend/src/app/components/playlist/mapper_script_view.rs @@ -123,21 +123,16 @@ fn render_null_value() -> Html { html! { { "null" } } } -fn render_map_case(case: &MapCase, script: &MapperScript, format_params: &mut FormatParams) -> Html { - let keys_html = html! { +fn render_joined_keys(keys: &[T], render_key: F) -> Html +where + F: Fn(&T) -> Html, +{ + html! { <> { - for case.keys.iter().enumerate().map(|(i, key)| { - let item = match key { - MapCaseKey::Text(text) => render_literal(text), - MapCaseKey::RangeFrom(from) => html! { {format!("{from}..")} }, - MapCaseKey::RangeTo(to) => html! { {format!("..{to}")} }, - MapCaseKey::RangeFull(from, to) => html! { {format!("{from}..{to}")} }, - MapCaseKey::RangeEq(val) => html! { {val.to_string()} }, - MapCaseKey::AnyMatch => html! { {"_"} }, - }; - - if i < case.keys.len() - 1 { + for keys.iter().enumerate().map(|(i, key)| { + let item = render_key(key); + if i < keys.len() - 1 { html! { <> { item } { ", " } } } else { html! { { item } } @@ -145,20 +140,40 @@ fn render_map_case(case: &MapCase, script: &MapperScript, format_params: &mut Fo }) } - }; - let has_bracket = case.keys.len() > 1; + } +} + +fn render_case_clause( + keys_html: Html, + has_bracket: bool, + expression_html: Html, + format_params: &mut FormatParams, +) -> Html { html! { <> {indent(format_params.level, true)} {if has_bracket {"("} else {""}} {keys_html} {if has_bracket {")"} else {""}} - {" => "} {render_expression(&case.expression, script, format_params)}{","} + {" => "} {expression_html}{","} {newline(format_params)} } } +fn render_map_case(case: &MapCase, script: &MapperScript, format_params: &mut FormatParams) -> Html { + let keys_html = render_joined_keys(&case.keys, |key| match key { + MapCaseKey::Text(text) => render_literal(text), + MapCaseKey::RangeFrom(from) => html! { {format!("{from}..")} }, + MapCaseKey::RangeTo(to) => html! { {format!("..{to}")} }, + MapCaseKey::RangeFull(from, to) => html! { {format!("{from}..{to}")} }, + MapCaseKey::RangeEq(val) => html! { {val.to_string()} }, + MapCaseKey::AnyMatch => html! { {"_"} }, + }); + let expr_html = render_expression(&case.expression, script, format_params); + render_case_clause(keys_html, case.keys.len() > 1, expr_html, format_params) +} + fn render_map_cases(cases: &[MapCase], script: &MapperScript, format_params: &mut FormatParams) -> Html { html! { <> @@ -266,35 +281,12 @@ fn render_assignment( } fn render_match_case(case: &MatchCase, script: &MapperScript, format_params: &mut FormatParams) -> Html { - let keys_html = html! { - <> - { - for case.keys.iter().enumerate().map(|(i, key)| { - let item = match key { - MatchCaseKey::Identifier(text) => render_identifier(text), - MatchCaseKey::AnyMatch => html! { {"_"} }, - }; - - if i < case.keys.len() - 1 { - html! { <> { item } { ", " } } - } else { - html! { { item } } - } - }) - } - - }; - let has_bracket = case.keys.len() > 1; - html! { - <> - {indent(format_params.level, true)} - {if has_bracket {"("} else {""}} - {keys_html} - {if has_bracket {")"} else {""}} - {" => "} {render_expression(&case.expression, script, format_params)}{","} - {newline(format_params)} - - } + let keys_html = render_joined_keys(&case.keys, |key| match key { + MatchCaseKey::Identifier(text) => render_identifier(text), + MatchCaseKey::AnyMatch => html! { {"_"} }, + }); + let expr_html = render_expression(&case.expression, script, format_params); + render_case_clause(keys_html, case.keys.len() > 1, expr_html, format_params) } fn render_match_cases(cases: &[MatchCase], script: &MapperScript, format_params: &mut FormatParams) -> Html { diff --git a/frontend/src/app/components/playlist/playlist_explorer.rs b/frontend/src/app/components/playlist/playlist_explorer.rs index fd122ef95..232c052c2 100644 --- a/frontend/src/app/components/playlist/playlist_explorer.rs +++ b/frontend/src/app/components/playlist/playlist_explorer.rs @@ -17,23 +17,17 @@ use crate::{ services::{CreateRecordingTaskRequest, DialogService, RecordingService, RecordingSourceInput}, }; use shared::{ - error::TuliproxError, model::{ Permission, PlaylistRequest, PlaylistUrlResolveRequest, SearchRequest, SeriesStreamDetailEpisodeProperties, SeriesStreamProperties, UiPlaylistGroup, UiPlaylistItem, VirtualId, XtreamCluster, }, utils::{format_float_localized, Internable}, }; -use std::{cell::RefCell, collections::HashMap, fmt::Display, rc::Rc, str::FromStr}; +use std::{cell::RefCell, collections::HashMap, rc::Rc, str::FromStr}; use wasm_bindgen::JsCast; use web_sys::HtmlInputElement; use yew::{platform::spawn_local, prelude::*}; -const COPY_LINK_TULIPROX_VIRTUAL_ID: &str = "copy_link_tuliprox_virtual_id"; -const COPY_LINK_TULIPROX_WEBPLAYER_URL: &str = "copy_link_tuliprox_webplayer_url"; -const COPY_LINK_PROVIDER_URL: &str = "copy_link_provider_url"; -const DOWNLOAD_ITEM: &str = "download_item"; -const RECORD_ITEM: &str = "record_item"; const TP_EXPLORER_SEARCH_FIELDS_KEY: &str = "tp-explorer-search-fields"; #[derive(Clone)] @@ -47,51 +41,18 @@ struct ChannelSelection { } #[allow(clippy::enum_variant_names)] -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, strum_macros::Display, strum_macros::EnumString)] +#[strum(serialize_all = "snake_case")] enum ExplorerAction { CopyLinkTuliproxVirtualId, CopyLinkTuliproxWebPlayerUrl, CopyLinkProviderUrl, + #[strum(serialize = "download_item")] Download, + #[strum(serialize = "record_item")] Record, } -impl Display for ExplorerAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::CopyLinkTuliproxVirtualId => COPY_LINK_TULIPROX_VIRTUAL_ID, - Self::CopyLinkTuliproxWebPlayerUrl => COPY_LINK_TULIPROX_WEBPLAYER_URL, - Self::CopyLinkProviderUrl => COPY_LINK_PROVIDER_URL, - Self::Download => DOWNLOAD_ITEM, - Self::Record => RECORD_ITEM, - } - ) - } -} - -impl FromStr for ExplorerAction { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - if s.eq(COPY_LINK_TULIPROX_VIRTUAL_ID) { - Ok(Self::CopyLinkTuliproxVirtualId) - } else if s.eq(COPY_LINK_TULIPROX_WEBPLAYER_URL) { - Ok(Self::CopyLinkTuliproxWebPlayerUrl) - } else if s.eq(COPY_LINK_PROVIDER_URL) { - Ok(Self::CopyLinkProviderUrl) - } else if s.eq(DOWNLOAD_ITEM) { - Ok(Self::Download) - } else if s.eq(RECORD_ITEM) { - Ok(Self::Record) - } else { - Err(TuliproxError::Config(format!("Unknown ExplorerAction: {s}"))) - } - } -} - fn build_download_filename(title: &str, url: &str) -> String { let sanitized = title .chars() diff --git a/frontend/src/app/components/playlist/target_table.rs b/frontend/src/app/components/playlist/target_table.rs index 2adca1a86..b6509f340 100644 --- a/frontend/src/app/components/playlist/target_table.rs +++ b/frontend/src/app/components/playlist/target_table.rs @@ -13,11 +13,8 @@ use crate::{ model::DialogResult, services::DialogService, }; -use shared::{ - error::TuliproxError, - model::{ConfigTargetDto, SortOrder}, -}; -use std::{fmt::Display, rc::Rc, str::FromStr}; +use shared::model::{ConfigTargetDto, SortOrder}; +use std::{rc::Rc, str::FromStr}; use yew::{platform::spawn_local, prelude::*}; const HEADERS: [&str; 12] = [ @@ -76,37 +73,59 @@ pub fn TargetTable(props: &TargetTableProps) -> Html { let translator = translate.clone(); let popup_onclick = handle_popup_onclick.clone(); Callback::<(usize, usize, Rc), Html>::from( - move |(row, col, dto): (usize, usize, Rc)| { - match col { - 0 => { - let popup_onclick = popup_onclick.clone(); - html! { - - } + move |(row, col, dto): (usize, usize, Rc)| match col { + 0 => { + let popup_onclick = popup_onclick.clone(); + html! { + } - 1 => html! { }, - 2 => html! { dto.name.as_str() }, - 3 => html! { }, - 4 => html! { }, - 5 => dto.sort.as_ref().map_or_else(|| html! {}, |_s| html! { }), - 6 => dto.t_filter.as_ref().map_or_else(|| html! {}, |f| html! { })}> }), - 7 => dto.rename.as_ref().map_or_else(|| html! {}, |_r| html! { }), - 8 => { - let mapping_oneliner = dto.mapping.as_ref().map(|v| v.join(", ")).unwrap_or_default(); - html_if!(!mapping_oneliner.is_empty(), - { }) - }, - 9 => html! { }, - 10 => html! { }, - 11 => html! { }, - _ => html! {""}, } + 1 => html! { }, + 2 => html! { dto.name.as_str() }, + 3 => html! { }, + 4 => { + html! { } + } + 5 => dto.sort.as_ref().map_or_else( + || html! {}, + |_s| html! { }, + ), + 6 => { + let filters = [ + (translator.t("LABEL.FILTER"), dto.filter.t_processing.as_ref()), + (translator.t("LABEL.PERSIST_FILTER"), dto.filter.t_persist.as_ref()), + ]; + let rendered = filters.into_iter().filter_map(|(label, filter)| { + filter.map(|filter| { + html! { +
+ {label} + +
+ } + }) + }); + html! { { for rendered } } + } + 7 => dto.rename.as_ref().map_or_else( + || html! {}, + |_r| html! { }, + ), + 8 => { + let mapping_oneliner = dto.mapping.as_ref().map(|v| v.join(", ")).unwrap_or_default(); + html_if!(!mapping_oneliner.is_empty(), + { }) + } + 9 => html! { }, + 10 => html! { }, + 11 => html! { }, + _ => html! {""}, }, ) }; @@ -216,36 +235,10 @@ pub fn TargetTable(props: &TargetTableProps) -> Html { } } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, strum_macros::Display, strum_macros::EnumString)] +#[strum(serialize_all = "snake_case")] enum TargetTableAction { Edit, Refresh, Delete, } - -impl Display for TargetTableAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Edit => "edit", - Self::Refresh => "refresh", - Self::Delete => "delete", - } - ) - } -} - -impl FromStr for TargetTableAction { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s { - "edit" => Ok(Self::Edit), - "refresh" => Ok(Self::Refresh), - "delete" => Ok(Self::Delete), - _ => Err(TuliproxError::Config(format!("Unknown Target Action: {s}"))), - } - } -} diff --git a/frontend/src/app/components/source_editor/editor_view.rs b/frontend/src/app/components/source_editor/editor_view.rs index c9ef3be4b..6e0b391aa 100644 --- a/frontend/src/app/components/source_editor/editor_view.rs +++ b/frontend/src/app/components/source_editor/editor_view.rs @@ -286,11 +286,7 @@ fn create_instance(block_type: BlockType) -> BlockInstance { BlockType::InputStalker => BlockInstance::Input(Rc::new(ConfigInputDto::new_with_type(InputType::Stalker))), BlockType::InputStaged => BlockInstance::Input(Rc::new(ConfigInputDto::new_with_type(InputType::Staged))), BlockType::Target => { - let dto = ConfigTargetDto { - name: String::new(), - filter: r#"Group ~ ".*""#.to_string(), - ..ConfigTargetDto::default() - }; + let dto = ConfigTargetDto { name: String::new(), ..ConfigTargetDto::default() }; BlockInstance::Target(Rc::new(dto)) } BlockType::OutputM3u => BlockInstance::Output(Rc::new(TargetOutputDto::M3u(M3uTargetOutputDto::default()))), diff --git a/frontend/src/app/components/source_editor/target_form.rs b/frontend/src/app/components/source_editor/target_form.rs index 32cc5f600..80f255950 100644 --- a/frontend/src/app/components/source_editor/target_form.rs +++ b/frontend/src/app/components/source_editor/target_form.rs @@ -10,7 +10,7 @@ use crate::{ }; use shared::{ error::TuliproxError, - model::{ClusterFlags, ConfigTargetDto, ConfigTargetOptions, ProcessingOrder}, + model::{ClusterFlags, ConfigTargetDto, ConfigTargetFilterDto, ConfigTargetOptions, ProcessingOrder}, utils::Internable, }; use std::{fmt::Display, rc::Rc, str::FromStr, sync::Arc}; @@ -22,6 +22,7 @@ use yew::{ const LABEL_ENABLED: &str = "LABEL.ENABLED"; const LABEL_NAME: &str = "LABEL.NAME"; const LABEL_FILTER: &str = "LABEL.FILTER"; +const LABEL_PERSIST_FILTER: &str = "LABEL.PERSIST_FILTER"; const LABEL_MAPPING: &str = "LABEL.MAPPING"; const LABEL_WATCH: &str = "LABEL.WATCH"; const LABEL_ADD_MAPPING: &str = "LABEL.ADD_MAPPING"; @@ -29,7 +30,7 @@ const LABEL_ADD_WATCH: &str = "LABEL.ADD_WATCH"; const LABEL_USE_MEMORY_CACHE: &str = "LABEL.USE_MEMORY_CACHE"; const LABEL_PROCESSING_ORDER: &str = "LABEL.PROCESSING_ORDER"; const LABEL_IGNORE_LOGO: &str = "LABEL.IGNORE_LOGO"; -const LABEL_REQUIRED_EPG: &str = "LABEL.REQUIRED_EPG"; +const LABEL_CLEAR_INVALID_EPG_IDS: &str = "LABEL.CLEAR_INVALID_EPG_IDS"; const LABEL_SHARE_LIVE_STREAMS: &str = "LABEL.SHARE_LIVE_STREAMS"; const LABEL_HLS: &str = "LABEL.HLS"; const LABEL_MPEG_TS: &str = "LABEL.MPEG_TS"; @@ -105,7 +106,7 @@ impl HasFormData for ConfigTargetOptionsFormState { #[derive(Clone)] pub enum ConfigTargetOptionsFormAction { IgnoreLogo(bool), - RequiredEpg(bool), + ClearInvalidEpgIds(bool), ShareLiveStreamsHls(bool), ShareLiveStreamsMpegTs(bool), RemoveDuplicates(bool), @@ -127,8 +128,8 @@ impl yew::prelude::Reducible for ConfigTargetOptionsFormState { form.ignore_logo = value; modified = true; } - ConfigTargetOptionsFormAction::RequiredEpg(value) => { - form.required_epg = value; + ConfigTargetOptionsFormAction::ClearInvalidEpgIds(value) => { + form.clear_invalid_epg_ids = value; modified = true; } ConfigTargetOptionsFormAction::ShareLiveStreamsHls(value) => { @@ -172,7 +173,7 @@ generate_form_reducer!( Enabled => enabled: bool, Name => name: String, ProcessingOrder => processing_order: ProcessingOrder, - Filter => filter: String, + Filter => filter: ConfigTargetFilterDto, Mapping => mapping: Option>, Watch => watch: Option>, UseMemoryCache => use_memory_cache: bool, @@ -313,7 +314,7 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html {
{ edit_field_bool!(target_options_state, translate.t(LABEL_IGNORE_LOGO), ignore_logo, ConfigTargetOptionsFormAction::IgnoreLogo) } - { edit_field_bool!(target_options_state, translate.t(LABEL_REQUIRED_EPG), required_epg, ConfigTargetOptionsFormAction::RequiredEpg) } + { edit_field_bool!(target_options_state, translate.t(LABEL_CLEAR_INVALID_EPG_IDS), clear_invalid_epg_ids, ConfigTargetOptionsFormAction::ClearInvalidEpgIds) }
{ translate.t(LABEL_SHARE_LIVE_STREAMS) } @@ -366,7 +367,7 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html {
{ config_field_bool!(target_options_state.form, translate.t(LABEL_IGNORE_LOGO), ignore_logo) } - { config_field_bool!(target_options_state.form, translate.t(LABEL_REQUIRED_EPG), required_epg) } + { config_field_bool!(target_options_state.form, translate.t(LABEL_CLEAR_INVALID_EPG_IDS), clear_invalid_epg_ids) }
{ translate.t(LABEL_SHARE_LIVE_STREAMS) } @@ -411,6 +412,7 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { let render_target = || { let target_form_state_1 = target_form_state.clone(); let target_form_state_2 = target_form_state.clone(); + let target_form_state_3 = target_form_state.clone(); if props.allow_write { html! { @@ -421,8 +423,19 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { { edit_field_text!(target_form_state, translate.t(LABEL_NAME), name, ConfigTargetFormAction::Name) } { config_field_child!(translate.t(LABEL_FILTER), "TARGET_FORM.FILTER", { html! { - | { - target_form_state_2.dispatch(ConfigTargetFormAction::Filter(new_filter.unwrap_or_default())); + | { + let mut filter = target_form_state_2.form.filter.clone(); + filter.processing = new_filter.filter(|value| !value.is_empty()); + target_form_state_2.dispatch(ConfigTargetFormAction::Filter(filter)); + })} /> + } + })} + { config_field_child!(translate.t(LABEL_PERSIST_FILTER), "TARGET_FORM.PERSIST_FILTER", { + html! { + | { + let mut filter = target_form_state_3.form.filter.clone(); + filter.persist = new_filter.filter(|value| !value.is_empty()); + target_form_state_3.dispatch(ConfigTargetFormAction::Filter(filter)); })} /> } })} @@ -453,7 +466,14 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { { config_field_bool!(target_form_state.form, translate.t(LABEL_USE_MEMORY_CACHE), use_memory_cache) }
{ config_field!(target_form_state.form, translate.t(LABEL_NAME), name) } - { config_field_custom!(translate.t(LABEL_FILTER), target_form_state.form.filter.clone()) } + { config_field_custom!( + translate.t(LABEL_FILTER), + target_form_state.form.filter.processing.clone().unwrap_or_default() + ) } + { config_field_custom!( + translate.t(LABEL_PERSIST_FILTER), + target_form_state.form.filter.persist.clone().unwrap_or_default() + ) } { config_field_custom!( translate.t(LABEL_PROCESSING_ORDER), target_form_state.form.processing_order.to_string() @@ -562,10 +582,10 @@ mod tests { } #[test] - fn required_epg_action_updates_target_option() { - let state = default_options_state().reduce(ConfigTargetOptionsFormAction::RequiredEpg(true)); + fn clear_invalid_epg_ids_action_updates_target_option() { + let state = default_options_state().reduce(ConfigTargetOptionsFormAction::ClearInvalidEpgIds(true)); - assert!(state.form.required_epg); + assert!(state.form.clear_invalid_epg_ids); assert!(state.modified); } diff --git a/frontend/src/app/components/userlist/user_table.rs b/frontend/src/app/components/userlist/user_table.rs index b098b9e47..dab38afd0 100644 --- a/frontend/src/app/components/userlist/user_table.rs +++ b/frontend/src/app/components/userlist/user_table.rs @@ -17,11 +17,10 @@ use crate::{ }; use shared::{ defaults::default_page_size, - error::TuliproxError, model::{permission::Permission, SortOrder}, utils::{unix_ts_to_str, Substring}, }; -use std::{cmp::Ordering, collections::HashSet, fmt::Display, rc::Rc, str::FromStr}; +use std::{cmp::Ordering, collections::HashSet, rc::Rc, str::FromStr}; use yew::{platform::spawn_local, prelude::*}; const HEADERS: [&str; 19] = [ @@ -66,7 +65,8 @@ fn get_cell_value(user: &TargetUser, col: usize) -> CellValue<'_> { fn is_col_sortable(col: usize) -> bool { matches!(col, 1 | 2 | 3 | 4 | 7 | 8 | 9 | 10 | 11 | 12 | 16 | 17) } -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, strum_macros::Display, strum_macros::EnumString)] +#[strum(serialize_all = "snake_case")] enum TableAction { Edit, Refresh, @@ -74,39 +74,6 @@ enum TableAction { CopyCredentials, } -impl Display for TableAction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Edit => "edit", - Self::Refresh => "refresh", - Self::Delete => "delete", - Self::CopyCredentials => "copy_credentials", - } - ) - } -} - -impl FromStr for TableAction { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - if s.eq("edit") { - Ok(Self::Edit) - } else if s.eq("refresh") { - Ok(Self::Refresh) - } else if s.eq("delete") { - Ok(Self::Delete) - } else if s.eq("copy_credentials") { - Ok(Self::CopyCredentials) - } else { - Err(TuliproxError::Config(format!("Unknown TableAction: {s}"))) - } - } -} - #[derive(Properties, PartialEq, Clone)] pub struct UserTableProps { pub users: TargetUserList, diff --git a/frontend/src/model/explorer_source_type.rs b/frontend/src/model/explorer_source_type.rs index db7e0b62f..e79eb7db1 100644 --- a/frontend/src/model/explorer_source_type.rs +++ b/frontend/src/model/explorer_source_type.rs @@ -1,11 +1,20 @@ -use shared::{error::TuliproxError, utils::Internable}; -use std::{fmt, str::FromStr, sync::Arc}; +use shared::utils::Internable; +use std::sync::Arc; -const HOSTED: &str = "hosted"; -const PROVIDER: &str = "provider"; -const CUSTOM: &str = "custom"; - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + strum_macros::Display, + strum_macros::EnumString, + strum_macros::IntoStaticStr, +)] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] pub enum ExplorerSourceType { Hosted, Provider, @@ -13,35 +22,15 @@ pub enum ExplorerSourceType { } impl ExplorerSourceType { - pub fn as_str(&self) -> &'static str { + pub const fn as_str(&self) -> &'static str { match self { - ExplorerSourceType::Hosted => HOSTED, - ExplorerSourceType::Provider => PROVIDER, - ExplorerSourceType::Custom => CUSTOM, + Self::Hosted => "hosted", + Self::Provider => "provider", + Self::Custom => "custom", } } } -impl FromStr for ExplorerSourceType { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "hosted" => Ok(ExplorerSourceType::Hosted), - "provider" => Ok(ExplorerSourceType::Provider), - "custom" => Ok(ExplorerSourceType::Custom), - _ => Err(TuliproxError::Config(format!("Unknown explorer source type: {s}"))), - } - } -} - -impl fmt::Display for ExplorerSourceType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = self.as_str(); - write!(f, "{s}") - } -} - impl Internable for ExplorerSourceType { fn intern(self) -> Arc { self.as_str().intern() } } diff --git a/frontend/src/services/config_service.rs b/frontend/src/services/config_service.rs index 50259ba1e..3f19c1506 100644 --- a/frontend/src/services/config_service.rs +++ b/frontend/src/services/config_service.rs @@ -208,7 +208,16 @@ impl ConfigService { for source in &mut app_config.sources.sources { for target in &mut source.targets { let prepared_templates = templates.as_deref(); - target.t_filter = get_filter(target.filter.as_str(), prepared_templates).ok(); + target.filter.t_processing = target.filter.processing.as_deref().and_then(|filter| { + get_filter(filter, prepared_templates) + .map_err(|e| error!("Failed to parse target processing filter: {e}")) + .ok() + }); + target.filter.t_persist = target.filter.persist.as_deref().and_then(|filter| { + get_filter(filter, prepared_templates) + .map_err(|e| error!("Failed to parse target persist filter: {e}")) + .ok() + }); if let Some(sort) = target.sort.as_mut() { for rule in &mut sort.rules { rule.t_filter = get_filter(&rule.filter, prepared_templates) diff --git a/shared/src/foundation/filter.pest b/shared/src/foundation/filter.pest index eb938ec16..7cf0d8cca 100644 --- a/shared/src/foundation/filter.pest +++ b/shared/src/foundation/filter.pest @@ -1,14 +1,25 @@ WHITESPACE = _{ " " | "\t" | "\r" | "\n"} -field = { ^"group" | ^"title" | ^"name" | ^"url" | ^"input" | ^"caption"} +field = { ^"group" | ^"title" | ^"name" | ^"genre" | ^"url" | ^"input" | ^"caption" | ^"epgid"} +numeric_field = { ^"chno" | ^"quality" } and = { ^"and" } or = { ^"or" } not = { ^"not" } regexp = @{ "\"" ~ ( "\\\"" | (!"\"" ~ ANY) )* ~ "\"" } +number = @{ ASCII_DIGIT+ } type_value = { ^"live" | ^"vod" | ^"movie" | ^"series" } type_comparison = { ^"type" ~ "=" ~ type_value } field_comparison_value = _{ regexp } field_comparison = { field ~ "~" ~ field_comparison_value } -comparison = { field_comparison | type_comparison } +str_op = { "!=" | "=" | ^"contains" | ^"startswith" } +string_comparison = { field ~ str_op ~ regexp } +presence_negation = { ^"not" } +presence_op = { (^"is" ~ presence_negation?) | "!=" | "=" } +presence_comparison = { field ~ presence_op ~ ^"empty" } +num_op = { ">=" | "<=" | "!=" | ">" | "<" | "=" } +numeric_comparison = { numeric_field ~ num_op ~ number } +set_values = { regexp ~ ("," ~ regexp)* } +set_comparison = { field ~ ^"in" ~ "[" ~ set_values ~ "]" } +comparison = { field_comparison | type_comparison | numeric_comparison | set_comparison | presence_comparison | string_comparison } bool_op = { and | or } expr_group = { "(" ~ expr ~ ")" } basic_expr = _{ comparison | expr_group } @@ -18,4 +29,4 @@ expr = { | basic_expr ~ (bool_op ~ expr)* } stmt = { expr ~ (bool_op ~ expr)* } -main = _{ SOI ~ stmt ~ EOI } \ No newline at end of file +main = _{ SOI ~ stmt ~ EOI } diff --git a/shared/src/foundation/filter.rs b/shared/src/foundation/filter.rs index 6d7298b51..d9b773edb 100644 --- a/shared/src/foundation/filter.rs +++ b/shared/src/foundation/filter.rs @@ -44,11 +44,14 @@ field_comparison_value = _{ regexp } field_comparison = { field ~ "~" ~ field_comparison_value } str_op = { "!=" | "=" | ^"contains" | ^"startswith" } string_comparison = { field ~ str_op ~ regexp } +presence_negation = { ^"not" } +presence_op = { (^"is" ~ presence_negation?) | "!=" | "=" } +presence_comparison = { field ~ presence_op ~ ^"empty" } num_op = { ">=" | "<=" | "!=" | ">" | "<" | "=" } numeric_comparison = { numeric_field ~ num_op ~ number } set_values = { regexp ~ ("," ~ regexp)* } set_comparison = { field ~ ^"in" ~ "[" ~ set_values ~ "]" } -comparison = { field_comparison | type_comparison | numeric_comparison | set_comparison | string_comparison } +comparison = { field_comparison | type_comparison | numeric_comparison | set_comparison | presence_comparison | string_comparison } bool_op = { and | or } expr_group = { "(" ~ expr ~ ")" } basic_expr = _{ comparison | expr_group } @@ -75,6 +78,21 @@ pub enum StringOperator { StartsWith, } +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum PresenceOperator { + IsEmpty, + IsNotEmpty, +} + +impl std::fmt::Display for PresenceOperator { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(match self { + Self::IsEmpty => "IS EMPTY", + Self::IsNotEmpty => "IS NOT EMPTY", + }) + } +} + impl std::fmt::Display for StringOperator { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.write_str(match *self { @@ -139,6 +157,7 @@ pub enum Filter { FieldComparison(ItemField, CompiledRegex), TypeComparison(ItemField, PlaylistItemType), StringComparison(ItemField, StringOperator, String), + PresenceComparison(ItemField, PresenceOperator), NumericComparison(ItemField, NumericOperator, u32), SetComparison(ItemField, Vec), UnaryExpression(UnaryOperator, Box), @@ -263,6 +282,13 @@ impl Filter { } is_match } + Self::PresenceComparison(field, op) => { + let is_empty = provider.get_filter_value(*field).is_none_or(|value| value.is_empty()); + match op { + PresenceOperator::IsEmpty => is_empty, + PresenceOperator::IsNotEmpty => !is_empty, + } + } Self::NumericComparison(field, op, value) => { let actual = match field { ItemField::Chno => provider.pli.header.chno, @@ -336,6 +362,7 @@ impl std::fmt::Display for Filter { Self::StringComparison(field, op, value) => { write!(f, "{field} {op} \"{}\"", value.replace('"', "\\\"")) } + Self::PresenceComparison(field, op) => write!(f, "{field} {op}"), Self::NumericComparison(field, op, value) => { write!(f, "{field} {op} {value}") } @@ -449,6 +476,24 @@ fn get_parser_string_comparison( Ok(Filter::StringComparison(field, op, value)) } +fn get_parser_presence_comparison(expr: Pair) -> Result { + let mut expr_inner = expr.into_inner(); + let field = get_parser_item_field( + &expr_inner + .next() + .ok_or_else(|| TuliproxError::FilterParse("presence comparison is missing a field".to_string()))?, + )?; + let operator = expr_inner + .next() + .ok_or_else(|| TuliproxError::FilterParse("presence comparison is missing an operator".to_string()))?; + let op = if operator.as_str() == "!=" || operator.as_str().split_whitespace().count() > 1 { + PresenceOperator::IsNotEmpty + } else { + PresenceOperator::IsEmpty + }; + Ok(Filter::PresenceComparison(field, op)) +} + fn get_parser_numeric_comparison(expr: Pair) -> Result { let mut expr_inner = expr.into_inner(); let field_pair = expr_inner.next().unwrap(); @@ -562,6 +607,10 @@ fn get_parser_expression( Ok(comp) => handle_expr!(bop, uop, stmts, comp), Err(err) => errors.push(err.to_string()), }, + Rule::presence_comparison => match get_parser_presence_comparison(pair) { + Ok(comp) => handle_expr!(bop, uop, stmts, comp), + Err(err) => errors.push(err.to_string()), + }, Rule::numeric_comparison => match get_parser_numeric_comparison(pair) { Ok(comp) => handle_expr!(bop, uop, stmts, comp), Err(err) => errors.push(err.to_string()), @@ -963,6 +1012,35 @@ mod tests { assert_filter_round_trip(r#"Group IN ["Sports", "News"] AND NOT (Name IN ["A"])"#); } + #[test] + fn test_filter_presence_round_trip_and_aliases() { + assert_filter_round_trip("EpgId IS EMPTY"); + assert_filter_round_trip("EpgId IS NOT EMPTY"); + + assert_eq!(get_filter("EpgId = EMPTY", None).expect("alias parses").to_string(), "EpgId IS EMPTY"); + assert_eq!(get_filter("EpgId != EMPTY", None).expect("alias parses").to_string(), "EpgId IS NOT EMPTY"); + } + + #[test] + fn test_filter_presence_eval() { + let empty = get_filter("EpgId IS EMPTY", None).expect("empty filter parses"); + let populated = get_filter("EpgId IS NOT EMPTY", None).expect("populated filter parses"); + let mut item = create_mock_pli("Channel", "Group"); + let provider = ValueProvider { pli: &item, match_as_ascii: false }; + assert!(empty.filter(&provider)); + assert!(!populated.filter(&provider)); + + item.header.epg_channel_id = Some("channel.epg".intern()); + let provider = ValueProvider { pli: &item, match_as_ascii: false }; + assert!(!empty.filter(&provider)); + assert!(populated.filter(&provider)); + + item.header.epg_channel_id = Some("".intern()); + let provider = ValueProvider { pli: &item, match_as_ascii: false }; + assert!(empty.filter(&provider)); + assert!(!populated.filter(&provider)); + } + #[test] fn test_filter_string_ops_eval() { let flt = r#"Group = "sports" AND Name CONTAINS "hd" AND NOT (Name STARTSWITH "x")"#; diff --git a/shared/src/model/auth_audit.rs b/shared/src/model/auth_audit.rs index a87d57513..eaeb14516 100644 --- a/shared/src/model/auth_audit.rs +++ b/shared/src/model/auth_audit.rs @@ -16,8 +16,20 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; /// What the auth layer decided. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Serialize, + Deserialize, + strum_macros::Display, + strum_macros::EnumString, + strum_macros::IntoStaticStr, +)] #[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] pub enum AuthAuditOutcome { /// Credentials verified and a token was issued. SignInSucceeded, diff --git a/shared/src/model/config/config_type.rs b/shared/src/model/config/config_type.rs index 600376db8..0feb30407 100644 --- a/shared/src/model/config/config_type.rs +++ b/shared/src/model/config/config_type.rs @@ -1,7 +1,6 @@ use serde::{Deserialize, Serialize}; -use std::fmt::{Display, Formatter}; -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, strum_macros::Display, strum_macros::EnumString)] pub enum ConfigType { Config, ApiProxy, @@ -9,19 +8,3 @@ pub enum ConfigType { Template, Sources, } - -impl Display for ConfigType { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Config => "Config", - Self::ApiProxy => "ApiProxy", - Self::Mapping => "Mapping", - Self::Template => "Template", - Self::Sources => "Sources", - } - ) - } -} diff --git a/shared/src/model/config/panel_api.rs b/shared/src/model/config/panel_api.rs index 750c9fb0f..07d7b7c53 100644 --- a/shared/src/model/config/panel_api.rs +++ b/shared/src/model/config/panel_api.rs @@ -11,8 +11,7 @@ use crate::{ }, }; use log::warn; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::{fmt, str::FromStr, sync::Arc}; +use std::sync::Arc; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default, PartialEq)] #[serde(deny_unknown_fields)] @@ -94,7 +93,11 @@ pub struct PanelApiAliasPoolDto { pub remove_expired: bool, } -#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +#[derive( + Default, Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, strum_macros::Display, strum_macros::EnumString, +)] +#[serde(rename_all = "UPPERCASE")] +#[strum(serialize_all = "UPPERCASE", ascii_case_insensitive)] pub enum PanelApiProvisioningMethod { #[default] Head, @@ -102,46 +105,13 @@ pub enum PanelApiProvisioningMethod { Post, } -impl fmt::Display for PanelApiProvisioningMethod { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - Self::Head => "HEAD", - Self::Get => "GET", - Self::Post => "POST", - }; - write!(f, "{s}") - } -} - -impl FromStr for PanelApiProvisioningMethod { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s.trim().to_ascii_uppercase().as_str() { - "HEAD" => Ok(Self::Head), - "GET" => Ok(Self::Get), - "POST" => Ok(Self::Post), - _ => Err(TuliproxError::Config(format!("Unknown provisioning method: {s}"))), - } - } -} - -impl Serialize for PanelApiProvisioningMethod { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> Deserialize<'de> for PanelApiProvisioningMethod { +impl<'de> serde::Deserialize<'de> for PanelApiProvisioningMethod { fn deserialize(deserializer: D) -> Result where - D: Deserializer<'de>, + D: serde::Deserializer<'de>, { - let s = String::deserialize(deserializer)?; - Self::from_str(&s).map_err(serde::de::Error::custom) + let value = ::deserialize(deserializer)?; + value.trim().parse().map_err(serde::de::Error::custom) } } @@ -246,3 +216,26 @@ impl PanelApiConfigDto { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::PanelApiProvisioningMethod; + + #[test] + fn provisioning_method_deserialization_remains_trimmed_and_case_insensitive() { + for (yaml, expected) in [ + ("head\n", PanelApiProvisioningMethod::Head), + ("' get '\n", PanelApiProvisioningMethod::Get), + ("Post\n", PanelApiProvisioningMethod::Post), + ] { + let parsed = serde_saphyr::from_str::(yaml).expect("method should parse"); + assert_eq!(parsed, expected); + } + } + + #[test] + fn provisioning_method_serializes_as_uppercase() { + let serialized = serde_saphyr::to_string(&PanelApiProvisioningMethod::Get).expect("method should serialize"); + assert_eq!(serialized.trim(), "GET"); + } +} diff --git a/shared/src/model/config/source.rs b/shared/src/model/config/source.rs index d0677153f..634e6acba 100644 --- a/shared/src/model/config/source.rs +++ b/shared/src/model/config/source.rs @@ -4,13 +4,51 @@ use crate::{ model::{ config::target::ConfigTargetDto, ConfigInputDto, ConfigProviderDto, HdHomeRunDeviceOverview, PatternTemplate, }, - utils::{arc_str_vec_serde, Internable}, + utils::{arc_str_vec_serde, is_sanitize_sensitive_info_enabled, Internable}, }; use log::warn; -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; const MAX_STAGE_CHAIN_DEPTH: usize = 2; +#[derive(Clone, Copy)] +struct CredentialOwner<'a> { + kind: &'static str, + name: &'a str, + url: &'a str, +} + +fn sensitive_url_for_log(url: &str, sanitize: bool) -> &str { + if sanitize { + "***" + } else { + url + } +} + +fn duplicate_credentials_warning( + current: CredentialOwner<'_>, + previous: CredentialOwner<'_>, + sanitize: bool, +) -> String { + let current_url = sensitive_url_for_log(current.url, sanitize); + if current.url == previous.url { + format!( + "The {} '{}' uses the same URL and credentials as the {} '{}' (URL: '{current_url}', username: '***', password: '***'). Tuliprox tracks provider connection limits separately for each input or alias, so the provider's actual connection limit may be exceeded. Reuse the existing provider account definition across multiple targets instead of defining it twice.", + current.kind, current.name, previous.kind, previous.name + ) + } else { + let previous_url = sensitive_url_for_log(previous.url, sanitize); + format!( + "The {} '{}' uses the same credentials as the {} '{}', but their URLs differ (URLs: '{current_url}' and '{previous_url}', username: '***', password: '***'). Tuliprox cannot determine whether both URLs point to the same provider account. If they do, connection limits are tracked separately for each input or alias, so the provider's actual connection limit may be exceeded.", + current.kind, current.name, previous.kind, previous.name + ) + } +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct ConfigSourceDto { @@ -91,23 +129,36 @@ impl SourcesConfigDto { let mut source_index: u16 = 0; let mut input_index: u16 = 0; let mut target_index: u16 = 1; - let mut input_credentials = HashSet::new(); + let mut input_credentials = HashMap::new(); // Prepare global inputs for input in &mut self.inputs { input_index = input.prepare(input_index, include_computed, provider_names, prepared_templates)?; if let (Some(username), Some(password)) = (input.username.as_ref(), input.password.as_ref()) { - let key = (username, password); - if !input_credentials.insert(key) { - warn!("Duplicate credentials found for input: '{}'", input.name); + let key = (username.as_str(), password.as_str()); + let current = CredentialOwner { kind: "input", name: input.name.as_ref(), url: input.url.as_str() }; + if let Some(previous) = input_credentials.get(&key) { + warn!( + "{}", + duplicate_credentials_warning(current, *previous, is_sanitize_sensitive_info_enabled()) + ); + } else { + input_credentials.insert(key, current); } } if let Some(aliases) = &input.aliases { for alias in aliases { if let (Some(username), Some(password)) = (alias.username.as_ref(), alias.password.as_ref()) { - let key = (username, password); - if !input_credentials.insert(key) { - warn!("Duplicate credentials found for input alias: '{}'", alias.name); + let key = (username.as_str(), password.as_str()); + let current = + CredentialOwner { kind: "input alias", name: alias.name.as_ref(), url: alias.url.as_str() }; + if let Some(previous) = input_credentials.get(&key) { + warn!( + "{}", + duplicate_credentials_warning(current, *previous, is_sanitize_sensitive_info_enabled()) + ); + } else { + input_credentials.insert(key, current); } } } @@ -235,6 +286,49 @@ mod tests { } } + #[test] + fn duplicate_credentials_warning_identifies_same_url_account() { + let previous = CredentialOwner { kind: "input", name: "primary", url: "provider://example" }; + let current = CredentialOwner { kind: "input", name: "duplicate", url: "provider://example" }; + + let warning = duplicate_credentials_warning(current, previous, false); + + assert!(warning.contains("same URL and credentials"), "Warning: {warning}"); + assert!(warning.contains("'duplicate'"), "Warning: {warning}"); + assert!(warning.contains("'primary'"), "Warning: {warning}"); + assert!(warning.contains("tracks provider connection limits separately"), "Warning: {warning}"); + assert!(warning.contains("provider://example"), "Warning: {warning}"); + assert!(warning.contains("username: '***', password: '***'"), "Warning: {warning}"); + } + + #[test] + fn duplicate_credentials_warning_explains_ambiguous_different_urls() { + let previous = CredentialOwner { kind: "input", name: "primary", url: "https://one.example" }; + let current = CredentialOwner { kind: "input alias", name: "possible-duplicate", url: "https://two.example" }; + + let warning = duplicate_credentials_warning(current, previous, false); + + assert!(warning.contains("same credentials"), "Warning: {warning}"); + assert!(warning.contains("URLs differ"), "Warning: {warning}"); + assert!(warning.contains("cannot determine whether both URLs point to the same provider account")); + assert!(warning.contains("connection limits are tracked separately"), "Warning: {warning}"); + assert!(warning.contains("https://one.example"), "Warning: {warning}"); + assert!(warning.contains("https://two.example"), "Warning: {warning}"); + assert!(warning.contains("username: '***', password: '***'"), "Warning: {warning}"); + } + + #[test] + fn duplicate_credentials_warning_masks_urls_when_sanitizing_logs() { + let previous = CredentialOwner { kind: "input", name: "primary", url: "provider://one" }; + let current = CredentialOwner { kind: "input", name: "possible-duplicate", url: "provider://two" }; + + let warning = duplicate_credentials_warning(current, previous, true); + + assert!(warning.contains("URLs: '***' and '***'"), "Warning: {warning}"); + assert!(warning.contains("username: '***', password: '***'"), "Warning: {warning}"); + assert!(!warning.contains("provider://"), "Warning must not expose either URL: {warning}"); + } + #[test] fn duplicate_default_target_names_are_rejected() { let sources = SourcesConfigDto { @@ -295,7 +389,7 @@ mod tests { inputs: vec!["input_1".intern()], targets: vec![ConfigTargetDto { name: "target_1".to_string(), - filter: "!FILTER_NAME!".to_string(), + filter: "!FILTER_NAME!".into(), output: vec![TargetOutputDto::M3u(M3uTargetOutputDto::default())], rename: Some(vec![ConfigRenameDto { field: ItemField::Name, @@ -425,7 +519,7 @@ mod tests { sources: vec![ConfigSourceDto { inputs: vec!["staged".intern()], targets: vec![ConfigTargetDto { - filter: r#"name ~ ".*""#.to_string(), + filter: r#"name ~ ".*""#.into(), output: vec![TargetOutputDto::M3u(M3uTargetOutputDto::default())], ..Default::default() }], diff --git a/shared/src/model/config/stream.rs b/shared/src/model/config/stream.rs index 582bb6ce4..7f5a8a052 100644 --- a/shared/src/model/config/stream.rs +++ b/shared/src/model/config/stream.rs @@ -10,27 +10,30 @@ use crate::{ error::TuliproxError, utils::{is_blank_optional_string, parse_to_kbps}, }; -use std::{ - fmt::{Display, Formatter}, - str::FromStr, -}; const STREAM_QUEUE_SIZE: usize = 1024; // mpsc channel holding messages. with 8192byte chunks and 2Mbit/s approx 8MB const MIN_SHARED_BURST_BUFFER_MB: u64 = 1; -#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)] +#[derive( + Debug, + Copy, + Clone, + serde::Serialize, + serde::Deserialize, + PartialEq, + Eq, + Hash, + strum_macros::Display, + strum_macros::EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] pub enum AdmissionStrategy { - #[serde(rename = "evict_user_same_ip_oldest")] EvictUserSameIpOldest, - #[serde(rename = "evict_user_same_ip_latest")] EvictUserSameIpLatest, - #[serde(rename = "evict_user_oldest")] EvictUserOldest, - #[serde(rename = "evict_user_latest")] EvictUserLatest, - #[serde(rename = "grace_instant_stream")] GraceInstantStream, - #[serde(rename = "grace_hold_stream")] GraceHoldStream, } @@ -40,39 +43,6 @@ impl AdmissionStrategy { pub fn is_grace_hold(&self) -> bool { matches!(self, Self::GraceHoldStream) } } -impl Display for AdmissionStrategy { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - AdmissionStrategy::EvictUserSameIpOldest => "evict_user_same_ip_oldest", - AdmissionStrategy::EvictUserSameIpLatest => "evict_user_same_ip_latest", - AdmissionStrategy::EvictUserOldest => "evict_user_oldest", - AdmissionStrategy::EvictUserLatest => "evict_user_latest", - AdmissionStrategy::GraceInstantStream => "grace_instant_stream", - AdmissionStrategy::GraceHoldStream => "grace_hold_stream", - } - ) - } -} - -impl FromStr for AdmissionStrategy { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - match s.trim() { - "evict_user_same_ip_oldest" => Ok(AdmissionStrategy::EvictUserSameIpOldest), - "evict_user_same_ip_latest" => Ok(AdmissionStrategy::EvictUserSameIpLatest), - "evict_user_oldest" => Ok(AdmissionStrategy::EvictUserOldest), - "evict_user_latest" => Ok(AdmissionStrategy::EvictUserLatest), - "grace_instant_stream" => Ok(AdmissionStrategy::GraceInstantStream), - "grace_hold_stream" => Ok(AdmissionStrategy::GraceHoldStream), - _ => Err(TuliproxError::Config(format!("Unknown admission strategy: {s}"))), - } - } -} - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StreamBufferConfigDto { diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index 9bb80c0f9..02ebd888d 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -12,6 +12,102 @@ use crate::{ utils::is_blank_optional_string, }; +#[derive(Debug, Clone, Default)] +pub struct ConfigTargetFilterDto { + pub processing: Option, + pub persist: Option, + pub t_processing: Option, + pub t_persist: Option, +} + +impl PartialEq for ConfigTargetFilterDto { + fn eq(&self, other: &Self) -> bool { self.processing == other.processing && self.persist == other.persist } +} + +impl From for ConfigTargetFilterDto { + fn from(processing: String) -> Self { Self { processing: Some(processing), ..Self::default() } } +} + +impl From<&str> for ConfigTargetFilterDto { + fn from(processing: &str) -> Self { processing.to_string().into() } +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct StagedTargetFilterDto { + #[serde(default)] + processing: Option, + #[serde(default)] + persist: Option, +} + +#[derive(serde::Deserialize)] +#[serde(untagged)] +enum TargetFilterRepr { + Processing(String), + Staged(StagedTargetFilterDto), +} + +impl<'de> serde::Deserialize<'de> for ConfigTargetFilterDto { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + match ::deserialize(deserializer)? { + TargetFilterRepr::Processing(processing) => Ok(processing.into()), + TargetFilterRepr::Staged(staged) => { + if staged.processing.is_none() && staged.persist.is_none() { + return Err(serde::de::Error::custom("staged target filter requires at least one stage")); + } + Ok(Self { processing: staged.processing, persist: staged.persist, ..Self::default() }) + } + } + } +} + +impl ConfigTargetFilterDto { + pub const fn is_empty(&self) -> bool { self.processing.is_none() && self.persist.is_none() } + + fn prepare(&mut self, templates: Option<&[PatternTemplate]>) -> Result<(), TuliproxError> { + fn compile_filter( + value: Option<&str>, + templates: Option<&[PatternTemplate]>, + ) -> Result, TuliproxError> { + value + .map(str::trim) + .filter(|filter| !filter.is_empty()) + .map(|filter| get_filter(filter, templates)) + .transpose() + } + + self.t_processing = compile_filter(self.processing.as_deref(), templates)?; + self.t_persist = compile_filter(self.persist.as_deref(), templates)?; + Ok(()) + } +} + +impl serde::Serialize for ConfigTargetFilterDto { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + + if self.persist.is_none() { + return serializer.serialize_str(self.processing.as_deref().unwrap_or_default()); + } + let field_count = usize::from(self.processing.is_some()) + usize::from(self.persist.is_some()); + let mut state = serializer.serialize_struct("ConfigTargetFilterDto", field_count)?; + if let Some(processing) = self.processing.as_ref() { + state.serialize_field("processing", processing)?; + } + if let Some(persist) = self.persist.as_ref() { + state.serialize_field("persist", persist)?; + } + state.end() + } +} + #[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct ConfigTargetShareLiveStreams { @@ -95,7 +191,8 @@ pub struct ConfigTargetOptions { #[serde(default, skip_serializing_if = "is_false")] pub ignore_logo: bool, #[serde(default, skip_serializing_if = "is_false")] - pub required_epg: bool, + #[serde(alias = "required_epg")] + pub clear_invalid_epg_ids: bool, #[serde( default, deserialize_with = "deserialize_share_live_streams", @@ -115,7 +212,7 @@ pub struct ConfigTargetOptions { impl ConfigTargetOptions { pub fn is_empty(&self) -> bool { !self.ignore_logo - && !self.required_epg + && !self.clear_invalid_epg_ids && self.share_live_streams.is_empty() && !self.remove_duplicates && self.deduplicate.is_none() @@ -127,7 +224,7 @@ impl ConfigTargetOptions { pub const fn lowercase_xmltv_display_names(&self) -> bool { self.epg_output.lowercase_xmltv_display_names } - pub const fn required_epg(&self) -> bool { self.required_epg } + pub const fn clear_invalid_epg_ids(&self) -> bool { self.clear_invalid_epg_ids } pub fn share_live_hls_enabled(&self) -> bool { self.share_live_streams.hls } @@ -315,7 +412,8 @@ pub struct ConfigTargetDto { pub options: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub sort: Option, - pub filter: String, + #[serde(default, skip_serializing_if = "ConfigTargetFilterDto::is_empty")] + pub filter: ConfigTargetFilterDto, #[serde(default)] pub output: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -330,8 +428,6 @@ pub struct ConfigTargetDto { pub watch: Option>, #[serde(default, skip_serializing_if = "is_false")] pub use_memory_cache: bool, - #[serde(skip)] - pub t_filter: Option, } impl Default for ConfigTargetDto { @@ -342,7 +438,7 @@ impl Default for ConfigTargetDto { name: default_as_default(), options: None, sort: None, - filter: String::new(), + filter: ConfigTargetFilterDto::default(), output: Vec::new(), rename: None, mapping: None, @@ -350,7 +446,6 @@ impl Default for ConfigTargetDto { processing_order: ProcessingOrder::default(), watch: None, use_memory_cache: false, - t_filter: None, } } } @@ -523,32 +618,26 @@ impl ConfigTargetDto { } } - match get_filter(&self.filter, templates) { - Ok(fltr) => { - // debug!("Filter: {}", fltr); - self.t_filter = Some(fltr); - self.rename.prepare_all(templates)?; - if let Some(sort) = self.sort.as_mut() { - sort.prepare(templates)?; - } - Ok(()) - } - Err(err) => Err(err), + self.filter.prepare(templates)?; + self.rename.prepare_all(templates)?; + if let Some(sort) = self.sort.as_mut() { + sort.prepare(templates)?; } + Ok(()) } } #[cfg(test)] mod tests { use super::{ - ConfigTargetDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, EpgOutputOptions, M3uTargetOutputDto, - StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto, + ConfigTargetDto, ConfigTargetFilterDto, ConfigTargetOptions, ConfigTargetShareLiveStreams, EpgOutputOptions, + M3uTargetOutputDto, StrmTargetOutputDto, TargetOutputDto, XtreamTargetOutputDto, }; fn target_with_outputs(output: Vec) -> ConfigTargetDto { ConfigTargetDto { name: "target".to_string(), - filter: "Group ~ \".*\"".to_string(), + filter: "Group ~ \".*\"".into(), output, ..ConfigTargetDto::default() } @@ -571,6 +660,125 @@ mod tests { fn xtream_output() -> TargetOutputDto { TargetOutputDto::Xtream(XtreamTargetOutputDto::default()) } + #[test] + fn target_filter_string_roundtrips_as_string() { + let dto = serde_saphyr::from_str::( + r#" +name: target +filter: 'Group ~ ".*"' +output: + - type: m3u +"#, + ) + .expect("legacy target filter should deserialize"); + + assert_eq!(dto.filter.processing.as_deref(), Some(r#"Group ~ ".*""#)); + assert_eq!(dto.filter.persist, None); + let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); + assert!(serialized.contains("filter: Group ~"), "expected scalar filter, got: {serialized}"); + assert!(!serialized.contains("processing:"), "processing-only filter must stay scalar: {serialized}"); + } + + #[test] + fn target_without_filter_defaults_to_no_stages_and_omits_filter() { + let dto = serde_saphyr::from_str::( + r" +name: target +output: + - type: m3u +", + ) + .expect("target without filter should deserialize"); + + assert!(dto.filter.is_empty()); + let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); + assert!(!serialized.contains("filter:"), "empty filter should be omitted: {serialized}"); + } + + #[test] + fn target_filter_stages_roundtrip_as_mapping() { + let dto = serde_saphyr::from_str::( + r#" +name: target +filter: + processing: 'Group ~ ".*"' + persist: 'EpgId ~ ".+"' +output: + - type: m3u +"#, + ) + .expect("staged target filter should deserialize"); + + assert_eq!(dto.filter.processing.as_deref(), Some(r#"Group ~ ".*""#)); + assert_eq!(dto.filter.persist.as_deref(), Some(r#"EpgId ~ ".+""#)); + let serialized = serde_saphyr::to_string(&dto).expect("target should serialize"); + assert!(serialized.contains("processing:"), "expected staged filter mapping: {serialized}"); + assert!(serialized.contains("persist:"), "expected persist filter: {serialized}"); + } + + #[test] + fn staged_target_filter_rejects_empty_mapping() { + let result = serde_saphyr::from_str::("{}\n"); + assert!(result.is_err()); + } + + #[test] + fn staged_target_filter_accepts_missing_processing_stage() { + let filter = serde_saphyr::from_str::( + r#"persist: 'EpgId ~ ".+"' +"#, + ) + .expect("persist-only filter should deserialize"); + + assert_eq!(filter.processing, None); + assert_eq!(filter.persist.as_deref(), Some(r#"EpgId ~ ".+""#)); + } + + #[test] + fn target_filter_treats_empty_and_whitespace_stages_as_match_all() { + for value in ["", " \t\r\n "] { + let mut filter = ConfigTargetFilterDto { + processing: Some(value.to_string()), + persist: Some(value.to_string()), + ..ConfigTargetFilterDto::default() + }; + + filter.prepare(None).expect("blank filter stages should prepare"); + + assert!(filter.t_processing.is_none()); + assert!(filter.t_persist.is_none()); + } + } + + #[test] + fn processing_only_mapping_serializes_as_scalar() { + let filter = serde_saphyr::from_str::( + r#"processing: 'Group ~ ".*"' +"#, + ) + .expect("processing-only mapping should deserialize"); + + let serialized = serde_saphyr::to_string(&filter).expect("processing filter should serialize"); + assert!(!serialized.contains("processing:"), "processing-only filter must become scalar: {serialized}"); + assert!(serialized.contains("Group ~")); + } + + #[test] + fn target_filter_persist_roundtrips_as_mapping() { + let filter = ConfigTargetFilterDto { + processing: None, + persist: Some(r#"EpgId ~ ".+""#.to_string()), + ..ConfigTargetFilterDto::default() + }; + + let serialized = serde_saphyr::to_string(&filter).expect("target filter should serialize"); + assert!(!serialized.contains("processing:")); + assert!(serialized.contains("persist:")); + let reparsed = + serde_saphyr::from_str::(&serialized).expect("target filter should deserialize"); + assert_eq!(reparsed, filter); + } + #[test] fn strm_with_username_is_allowed_with_m3u_output() { let mut target = @@ -659,7 +867,7 @@ share_live_streams: false let options = ConfigTargetOptions::default(); assert!(options.is_empty()); - assert!(!options.required_epg()); + assert!(!options.clear_invalid_epg_ids()); let serialized = serde_saphyr::to_string(&options).expect("default options should serialize"); assert!( @@ -683,15 +891,16 @@ share_live_streams: false } #[test] - fn target_options_required_epg_roundtrips_and_makes_options_nonempty() { + fn target_options_clear_invalid_epg_ids_roundtrips_and_accepts_legacy_alias() { let options = serde_saphyr::from_str::("required_epg: true\n") - .expect("required_epg should deserialize"); + .expect("legacy required_epg should deserialize"); - assert!(options.required_epg()); + assert!(options.clear_invalid_epg_ids()); assert!(!options.is_empty()); - let serialized = serde_saphyr::to_string(&options).expect("required_epg should serialize"); - assert!(serialized.contains("required_epg: true")); + let serialized = serde_saphyr::to_string(&options).expect("clear_invalid_epg_ids should serialize"); + assert!(serialized.contains("clear_invalid_epg_ids: true")); + assert!(!serialized.contains("required_epg:")); } #[test] diff --git a/shared/src/model/config/video_download.rs b/shared/src/model/config/video_download.rs index 61c67c7bc..f65d6a4e2 100644 --- a/shared/src/model/config/video_download.rs +++ b/shared/src/model/config/video_download.rs @@ -1158,46 +1158,51 @@ mod tests { assert!(err.to_string().contains("delete_after_days"), "error: {err}"); } + fn check_recording_disk_rejected(disk: RecordingDiskConfigDto, expected_substr: &str) { + let mut recording = make_recording_config(); + recording.disk = Some(disk); + let mut video = make_recording_video_config(recording); + let err = video.prepare().expect_err("invalid disk config should fail"); + assert!(err.to_string().contains(expected_substr), "error: {err}"); + } + #[test] fn recording_disk_high_water_above_100_is_rejected() { - let mut recording = make_recording_config(); - recording.disk = Some(RecordingDiskConfigDto { - high_water_percent: Some(101), - low_water_percent: None, - cleanup_interval_secs: Some(3600), - safety_bytes: Some(1024), - }); - let mut video = make_recording_video_config(recording); - let err = video.prepare().expect_err("high > 100 should fail"); - assert!(err.to_string().contains("high_water_percent"), "error: {err}"); + check_recording_disk_rejected( + RecordingDiskConfigDto { + high_water_percent: Some(101), + low_water_percent: None, + cleanup_interval_secs: Some(3600), + safety_bytes: Some(1024), + }, + "high_water_percent", + ); } #[test] fn recording_disk_low_water_above_100_is_rejected() { - let mut recording = make_recording_config(); - recording.disk = Some(RecordingDiskConfigDto { - high_water_percent: None, - low_water_percent: Some(101), - cleanup_interval_secs: Some(3600), - safety_bytes: Some(1024), - }); - let mut video = make_recording_video_config(recording); - let err = video.prepare().expect_err("low > 100 should fail"); - assert!(err.to_string().contains("low_water_percent"), "error: {err}"); + check_recording_disk_rejected( + RecordingDiskConfigDto { + high_water_percent: None, + low_water_percent: Some(101), + cleanup_interval_secs: Some(3600), + safety_bytes: Some(1024), + }, + "low_water_percent", + ); } #[test] fn recording_disk_low_ge_high_is_rejected() { - let mut recording = make_recording_config(); - recording.disk = Some(RecordingDiskConfigDto { - high_water_percent: Some(80), - low_water_percent: Some(80), - cleanup_interval_secs: Some(3600), - safety_bytes: Some(1024), - }); - let mut video = make_recording_video_config(recording); - let err = video.prepare().expect_err("low >= high should fail"); - assert!(err.to_string().contains("must be <"), "error: {err}"); + check_recording_disk_rejected( + RecordingDiskConfigDto { + high_water_percent: Some(80), + low_water_percent: Some(80), + cleanup_interval_secs: Some(3600), + safety_bytes: Some(1024), + }, + "must be <", + ); } #[test] @@ -1215,30 +1220,28 @@ mod tests { #[test] fn recording_disk_zero_cleanup_interval_is_rejected() { - let mut recording = make_recording_config(); - recording.disk = Some(RecordingDiskConfigDto { - high_water_percent: None, - low_water_percent: None, - cleanup_interval_secs: Some(0), - safety_bytes: None, - }); - let mut video = make_recording_video_config(recording); - let err = video.prepare().expect_err("zero cleanup_interval should fail"); - assert!(err.to_string().contains("cleanup_interval"), "error: {err}"); + check_recording_disk_rejected( + RecordingDiskConfigDto { + high_water_percent: None, + low_water_percent: None, + cleanup_interval_secs: Some(0), + safety_bytes: None, + }, + "cleanup_interval", + ); } #[test] fn recording_disk_zero_safety_bytes_is_rejected() { - let mut recording = make_recording_config(); - recording.disk = Some(RecordingDiskConfigDto { - high_water_percent: None, - low_water_percent: None, - cleanup_interval_secs: Some(3600), - safety_bytes: Some(0), - }); - let mut video = make_recording_video_config(recording); - let err = video.prepare().expect_err("zero safety_bytes should fail"); - assert!(err.to_string().contains("safety_bytes"), "error: {err}"); + check_recording_disk_rejected( + RecordingDiskConfigDto { + high_water_percent: None, + low_water_percent: None, + cleanup_interval_secs: Some(3600), + safety_bytes: Some(0), + }, + "safety_bytes", + ); } #[test] diff --git a/shared/src/model/epg.rs b/shared/src/model/epg.rs index 5f6ef8978..5c6ffbb96 100644 --- a/shared/src/model/epg.rs +++ b/shared/src/model/epg.rs @@ -186,18 +186,7 @@ impl EpgProgramme { impl EpgProgramme { pub fn new(start: i64, stop: i64, channel: Arc) -> Self { - Self { - start, - stop, - channel, - title: None, - desc: None, - catchup_id: None, - categories: Vec::new(), - is_live: false, - is_new: false, - previously_shown: false, - } + Self::new_all(start, stop, channel, None, None, None) } pub fn new_all( start: i64, diff --git a/shared/src/model/media_properties.rs b/shared/src/model/media_properties.rs index eed2e62c3..ca06c9cd0 100644 --- a/shared/src/model/media_properties.rs +++ b/shared/src/model/media_properties.rs @@ -1,17 +1,22 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use std::fmt; // Enum for Video Resolution -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default, strum_macros::Display)] pub enum VideoResolution { #[default] + #[strum(to_string = "")] Unknown, SD, + #[strum(to_string = "720p HD")] P720, + #[strum(to_string = "1080p FHD")] P1080, + #[strum(to_string = "1440p QHD")] P1440, + #[strum(to_string = "2160p 4K")] P2160, // 4K + #[strum(to_string = "4320p 8K")] P4320, // 8K } @@ -48,49 +53,27 @@ impl VideoResolution { } } -impl fmt::Display for VideoResolution { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - VideoResolution::SD => write!(f, "SD"), - VideoResolution::P720 => write!(f, "720p HD"), - VideoResolution::P1080 => write!(f, "1080p FHD"), - VideoResolution::P1440 => write!(f, "1440p QHD"), - VideoResolution::P2160 => write!(f, "2160p 4K"), - VideoResolution::P4320 => write!(f, "4320p 8K"), - VideoResolution::Unknown => write!(f, ""), - } - } -} - // Enum for Video Codec -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, strum_macros::Display)] pub enum VideoCodec { #[default] + #[strum(to_string = "")] Other, + #[strum(to_string = "H.264")] H264, + #[strum(to_string = "HEVC")] H265, MPEG4, + #[strum(to_string = "VC-1")] VC1, AV1, } -impl fmt::Display for VideoCodec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - VideoCodec::H264 => write!(f, "H.264"), // or AVC - VideoCodec::H265 => write!(f, "HEVC"), // or x265 - VideoCodec::MPEG4 => write!(f, "MPEG4"), - VideoCodec::VC1 => write!(f, "VC-1"), - VideoCodec::AV1 => write!(f, "AV1"), - VideoCodec::Other => write!(f, ""), - } - } -} - // Enum for Audio Codec -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, strum_macros::Display)] pub enum AudioCodec { #[default] + #[strum(to_string = "")] Other, AAC, AC3, @@ -100,47 +83,27 @@ pub enum AudioCodec { FLAC, } -impl fmt::Display for AudioCodec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - AudioCodec::AAC => write!(f, "AAC"), - AudioCodec::AC3 => write!(f, "AC3"), - AudioCodec::EAC3 => write!(f, "EAC3"), - AudioCodec::DTS => write!(f, "DTS"), - AudioCodec::TrueHD => write!(f, "TrueHD"), - AudioCodec::FLAC => write!(f, "FLAC"), - AudioCodec::Other => write!(f, ""), - } - } -} - // Enum for Audio Channels -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, strum_macros::Display)] pub enum AudioChannels { #[default] + #[strum(to_string = "")] Unknown, + #[strum(to_string = "1.0")] Mono, + #[strum(to_string = "2.0")] Stereo, + #[strum(to_string = "5.1")] Surround51, + #[strum(to_string = "7.1")] Surround71, } -impl fmt::Display for AudioChannels { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - AudioChannels::Mono => write!(f, "1.0"), - AudioChannels::Stereo => write!(f, "2.0"), - AudioChannels::Surround51 => write!(f, "5.1"), - AudioChannels::Surround71 => write!(f, "7.1"), - AudioChannels::Unknown => write!(f, ""), - } - } -} - // Enum for Video Dynamic Range -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, strum_macros::Display)] pub enum VideoDynamicRange { #[default] + #[strum(to_string = "")] SDR, HDR, HDR10, @@ -148,34 +111,15 @@ pub enum VideoDynamicRange { DV, // Dolby Vision } -impl fmt::Display for VideoDynamicRange { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - VideoDynamicRange::SDR => write!(f, ""), // Don't explicitly tag SDR - VideoDynamicRange::HDR => write!(f, "HDR"), - VideoDynamicRange::HDR10 => write!(f, "HDR10"), - VideoDynamicRange::HLG => write!(f, "HLG"), - VideoDynamicRange::DV => write!(f, "DV"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, strum_macros::Display)] pub enum VideoBitDepth { #[default] + #[strum(to_string = "")] Eight, + #[strum(to_string = "10bit")] Ten, } -impl fmt::Display for VideoBitDepth { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - VideoBitDepth::Eight => write!(f, ""), - VideoBitDepth::Ten => write!(f, "10bit"), - } - } -} - /// A struct that holds all classified media quality features. #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct MediaQuality { diff --git a/shared/src/model/messaging.rs b/shared/src/model/messaging.rs index 9e228c020..cfd40ab52 100644 --- a/shared/src/model/messaging.rs +++ b/shared/src/model/messaging.rs @@ -1,26 +1,35 @@ -use crate::{concat_string, error::TuliproxError}; -use std::{fmt, str::FromStr}; +use crate::concat_string; -#[derive(Debug, Copy, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)] +#[derive( + Debug, + Copy, + Clone, + serde::Serialize, + serde::Deserialize, + PartialEq, + Eq, + Hash, + strum_macros::Display, + strum_macros::EnumString, + strum_macros::IntoStaticStr, +)] +#[serde(rename_all = "snake_case")] +#[strum(ascii_case_insensitive)] pub enum MsgKind { - #[serde(rename = "info")] Info, - #[serde(rename = "stats")] Stats, - #[serde(rename = "error")] Error, - #[serde(rename = "watch")] Watch, - #[serde(rename = "disk_alert")] + #[strum(serialize = "DiskAlert", serialize = "disk_alert", serialize = "diskalert")] DiskAlert, /// A recording started. - #[serde(rename = "recording_started")] + #[strum(serialize = "RecordingStarted", serialize = "recording_started", serialize = "recordingstarted")] RecordingStarted, /// A recording completed. - #[serde(rename = "recording_completed")] + #[strum(serialize = "RecordingCompleted", serialize = "recording_completed", serialize = "recordingcompleted")] RecordingCompleted, /// A recording failed. - #[serde(rename = "recording_failed")] + #[strum(serialize = "RecordingFailed", serialize = "recording_failed", serialize = "recordingfailed")] RecordingFailed, } impl MsgKind { @@ -28,7 +37,7 @@ impl MsgKind { /// annotation on the variant and is used for config keys, template /// filenames, and any other text format that needs a stable identifier /// independent of the Rust variant name. - pub fn wire_name(&self) -> &'static str { + pub const fn wire_name(&self) -> &'static str { match self { MsgKind::Info => "info", MsgKind::Stats => "stats", @@ -49,50 +58,6 @@ impl MsgKind { } } -impl fmt::Display for MsgKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - MsgKind::Info => "Info", - MsgKind::Stats => "Stats", - MsgKind::Error => "Error", - MsgKind::Watch => "Watch", - MsgKind::DiskAlert => "DiskAlert", - MsgKind::RecordingStarted => "RecordingStarted", - MsgKind::RecordingCompleted => "RecordingCompleted", - MsgKind::RecordingFailed => "RecordingFailed", - }; - write!(f, "{s}") - } -} - -impl FromStr for MsgKind { - type Err = TuliproxError; - - fn from_str(s: &str) -> Result { - // Accepts both the snake_case wire name and the CamelCase variant - // name so values produced by `Display`/`to_string` round-trip. - if s.eq_ignore_ascii_case("info") { - Ok(Self::Info) - } else if s.eq_ignore_ascii_case("stats") { - Ok(Self::Stats) - } else if s.eq_ignore_ascii_case("error") { - Ok(Self::Error) - } else if s.eq_ignore_ascii_case("watch") { - Ok(Self::Watch) - } else if s.eq_ignore_ascii_case("disk_alert") || s.eq_ignore_ascii_case("diskalert") { - Ok(Self::DiskAlert) - } else if s.eq_ignore_ascii_case("recording_started") || s.eq_ignore_ascii_case("recordingstarted") { - Ok(Self::RecordingStarted) - } else if s.eq_ignore_ascii_case("recording_completed") || s.eq_ignore_ascii_case("recordingcompleted") { - Ok(Self::RecordingCompleted) - } else if s.eq_ignore_ascii_case("recording_failed") || s.eq_ignore_ascii_case("recordingfailed") { - Ok(Self::RecordingFailed) - } else { - Err(TuliproxError::Config(format!("Unknown MsgKind: {s}"))) - } - } -} - #[cfg(test)] mod tests { use super::MsgKind; diff --git a/shared/src/model/notification.rs b/shared/src/model/notification.rs index 8d37e43d2..ce4cabe52 100644 --- a/shared/src/model/notification.rs +++ b/shared/src/model/notification.rs @@ -101,8 +101,24 @@ impl<'de> serde::Deserialize<'de> for EventId { /// /// Ordered, so a channel can subscribe with `min_severity` and get /// everything at or above it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Default, + serde::Serialize, + serde::Deserialize, + strum_macros::Display, + strum_macros::EnumString, + strum_macros::IntoStaticStr, +)] #[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] pub enum Severity { /// Something finished normally. Safe to route nowhere. #[default] @@ -127,15 +143,7 @@ impl Severity { } #[must_use] - pub fn from_wire(s: &str) -> Option { - [Self::Info, Self::Warn, Self::Error, Self::Critical] - .into_iter() - .find(|c| c.wire_name().eq_ignore_ascii_case(s)) - } -} - -impl fmt::Display for Severity { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.wire_name()) } + pub fn from_wire(s: &str) -> Option { s.parse().ok() } } /// One entry in the event registry. diff --git a/shared/src/model/playlist_document.rs b/shared/src/model/playlist_document.rs index 0553514dd..962098756 100644 --- a/shared/src/model/playlist_document.rs +++ b/shared/src/model/playlist_document.rs @@ -6,7 +6,8 @@ use crate::{ XtreamVideoInfoDoc, XtreamVideoMovieData, }, utils::{ - arc_str_option_null_if_empty_serde, arc_str_serde, arc_str_vec_serde, extract_extension_from_url, Internable, + arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_serde, arc_str_vec_serde, + extract_extension_from_url, Internable, }, }; use indexmap::IndexMap; @@ -67,7 +68,7 @@ pub struct XtreamVideoDoc { #[serde(with = "arc_str_serde")] pub category_id: Arc, pub category_ids: Vec, - #[serde(with = "arc_str_serde")] + #[serde(with = "arc_str_null_is_none_serde")] pub container_extension: Arc, #[serde(default, with = "arc_str_option_null_if_empty_serde")] pub custom_sid: Option>, diff --git a/shared/src/model/playlist_info_document.rs b/shared/src/model/playlist_info_document.rs index 0be9928ff..f6f505db1 100644 --- a/shared/src/model/playlist_info_document.rs +++ b/shared/src/model/playlist_info_document.rs @@ -5,7 +5,10 @@ use crate::{ SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, VideoStreamProperties, VirtualId, XtreamCluster, XtreamMappingFlags, XtreamMappingOptions, }, - utils::{arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, arc_str_vec_serde, Internable}, + utils::{ + arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, + arc_str_vec_serde, Internable, + }, }; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; @@ -107,7 +110,7 @@ pub struct XtreamVideoMovieData { #[serde(with = "arc_str_serde")] pub category_id: Arc, pub category_ids: Vec, - #[serde(with = "arc_str_serde")] + #[serde(with = "arc_str_null_is_none_serde")] pub container_extension: Arc, #[serde(default, with = "arc_str_option_null_if_empty_serde")] pub custom_sid: Option>, @@ -191,7 +194,7 @@ pub struct XtreamSeriesEpisodeInfoDoc { pub episode_num: u32, #[serde(with = "arc_str_serde")] pub title: Arc, - #[serde(with = "arc_str_serde")] + #[serde(with = "arc_str_null_is_none_serde")] pub container_extension: Arc, pub info: XtreamSeriesEpisodeInfoData, #[serde(default, with = "arc_str_option_null_if_empty_serde")] diff --git a/shared/src/model/stalker_item.rs b/shared/src/model/stalker_item.rs index 1ff6b367a..cb278f55b 100644 --- a/shared/src/model/stalker_item.rs +++ b/shared/src/model/stalker_item.rs @@ -1,6 +1,6 @@ use crate::{ model::stalker::{StalkerPlaybackDescriptorDto, StalkerPortalCapabilitiesDto, StalkerStreamKind}, - utils::{arc_str_serde, arc_str_vec_serde, Internable}, + utils::{arc_str_null_is_none_option_serde, arc_str_serde, arc_str_vec_serde, Internable}, }; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -37,7 +37,7 @@ pub struct StalkerPlaylistItem { /// Raw `cmd` string returned by the portal (`ffmpeg `). #[serde(with = "arc_str_serde")] pub cmd: Arc, - #[serde(default)] + #[serde(default, with = "arc_str_null_is_none_option_serde")] pub container_extension: Option>, #[serde(default)] pub plot: Option>, @@ -181,7 +181,7 @@ pub struct StalkerEpisodeIndex { pub episode_number: i32, #[serde(with = "arc_str_serde")] pub title: Arc, - #[serde(default)] + #[serde(default, with = "arc_str_null_is_none_option_serde")] pub container_extension: Option>, #[serde(default)] pub added_at: i64, diff --git a/shared/src/model/stream_history_record.rs b/shared/src/model/stream_history_record.rs index fc15b8602..759a130e0 100644 --- a/shared/src/model/stream_history_record.rs +++ b/shared/src/model/stream_history_record.rs @@ -72,8 +72,11 @@ impl Display for DisconnectReason { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, strum_macros::Display, strum_macros::EnumString, +)] #[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] pub enum ConnectFailureReason { UserAccountExpired, UserConnectionsExhausted, @@ -86,22 +89,6 @@ pub enum ConnectFailureReason { Provisioning, } -impl Display for ConnectFailureReason { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - ConnectFailureReason::UserAccountExpired => write!(f, "user_account_expired"), - ConnectFailureReason::UserConnectionsExhausted => write!(f, "user_connections_exhausted"), - ConnectFailureReason::ProviderConnectionsExhausted => write!(f, "provider_connections_exhausted"), - ConnectFailureReason::ProviderError => write!(f, "provider_error"), - ConnectFailureReason::ProviderClosed => write!(f, "provider_closed"), - ConnectFailureReason::ChannelUnavailable => write!(f, "channel_unavailable"), - ConnectFailureReason::Preempted => write!(f, "preempted"), - ConnectFailureReason::SessionExpired => write!(f, "session_expired"), - ConnectFailureReason::Provisioning => write!(f, "provisioning"), - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FailureStage { diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index 5ad3386f3..72debdf2a 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -1,7 +1,7 @@ use crate::{ model::{info_doc_utils::InfoDocUtils, PlaylistEntry, XtreamSeriesInfo, XtreamSeriesInfoDoc, XtreamVideoInfo}, utils::{ - arc_str_default_on_null, arc_str_none_default_on_null, arc_str_option_null_if_empty_serde, + arc_str_none_default_on_null, arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_option_serde, deserialize_as_option_arc_str, deserialize_as_string_array, deserialize_json_as_opt_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, serialize_json_as_opt_string, Internable, CONSTANTS, @@ -291,7 +291,7 @@ pub struct VideoStreamProperties { pub custom_sid: Option>, #[serde(default, deserialize_with = "arc_str_none_default_on_null")] pub added: Arc, - #[serde(default, deserialize_with = "arc_str_default_on_null")] + #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] pub container_extension: Arc, #[serde(default, deserialize_with = "deserialize_number_from_string")] pub rating: Option, @@ -341,7 +341,7 @@ pub struct SeriesStreamDetailEpisodeProperties { pub season: u32, #[serde(default, deserialize_with = "arc_str_none_default_on_null")] pub title: Arc, - #[serde(default, deserialize_with = "arc_str_default_on_null")] + #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] pub container_extension: Arc, #[serde(default, with = "arc_str_option_null_if_empty_serde")] pub custom_sid: Option>, @@ -463,7 +463,7 @@ pub struct EpisodeStreamProperties { pub tmdb: Option, #[serde(default, deserialize_with = "arc_str_none_default_on_null")] pub movie_image: Arc, - #[serde(default, deserialize_with = "arc_str_default_on_null")] + #[serde(default, deserialize_with = "arc_str_null_is_none_serde::deserialize")] pub container_extension: Arc, #[serde( default, diff --git a/shared/src/model/xtream.rs b/shared/src/model/xtream.rs index e0a4cff17..5ddba2862 100644 --- a/shared/src/model/xtream.rs +++ b/shared/src/model/xtream.rs @@ -1,9 +1,9 @@ use crate::{ model::{ConfigProviderDto, ProxyUserStatus}, utils::{ - arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, deserialize_as_string_array, - deserialize_json_as_opt_string, deserialize_number_from_string, deserialize_number_from_string_or_zero, - serialize_json_as_opt_string, + arc_str_null_is_none_serde, arc_str_option_null_if_empty_serde, arc_str_option_serde, arc_str_serde, + deserialize_as_string_array, deserialize_json_as_opt_string, deserialize_number_from_string, + deserialize_number_from_string_or_zero, serialize_json_as_opt_string, }, }; use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer}; @@ -49,7 +49,7 @@ pub struct XtreamVideoInfoMovieData { pub custom_sid: Option>, #[serde(default, with = "arc_str_serde")] pub added: Arc, - #[serde(default, with = "arc_str_serde")] + #[serde(default, with = "arc_str_null_is_none_serde")] pub container_extension: Arc, } @@ -230,7 +230,7 @@ pub struct XtreamSeriesInfoEpisode { pub episode_num: u32, #[serde(default, with = "arc_str_serde")] pub title: Arc, - #[serde(default, with = "arc_str_serde")] + #[serde(default, with = "arc_str_null_is_none_serde")] pub container_extension: Arc, #[serde(default)] pub info: Option, diff --git a/shared/src/utils/hash_utils.rs b/shared/src/utils/hash_utils.rs index 97f99c53a..048a43a84 100644 --- a/shared/src/utils/hash_utils.rs +++ b/shared/src/utils/hash_utils.rs @@ -18,12 +18,12 @@ pub fn short_hash(text: &str) -> String { pub fn hex_encode(bytes: &[u8]) -> String { hex::encode_upper(bytes) } #[inline] -fn hex_nibble(b: u8) -> Result { +pub fn hex_digit(b: u8) -> Option { match b { - b'0'..=b'9' => Ok(b - b'0'), - b'a'..=b'f' => Ok(b - b'a' + 10), - b'A'..=b'F' => Ok(b - b'A' + 10), - _ => Err(()), + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, } } @@ -36,8 +36,8 @@ pub fn hex_decode(hex_str: &str) -> Result, String> { let mut out = Vec::with_capacity(bytes.len() / 2); let mut i = 0; while i < bytes.len() { - let hi = hex_nibble(bytes[i]).map_err(|()| format!("invalid hex at position {i}"))?; - let lo = hex_nibble(bytes[i + 1]).map_err(|()| format!("invalid hex at position {}", i + 1))?; + let hi = hex_digit(bytes[i]).ok_or_else(|| format!("invalid hex at position {i}"))?; + let lo = hex_digit(bytes[i + 1]).ok_or_else(|| format!("invalid hex at position {}", i + 1))?; out.push((hi << 4) | lo); i += 2; } @@ -189,8 +189,8 @@ pub fn parse_uuid_hex(s: &str) -> Option<[u8; 16]> { if di >= 16 || si + 1 >= src.len() { return None; } - let hi = hex_nibble(src[si]).ok()?; - let lo = hex_nibble(src[si + 1]).ok()?; + let hi = hex_digit(src[si])?; + let lo = hex_digit(src[si + 1])?; out[di] = (hi << 4) | lo; si += 2; di += 1; diff --git a/shared/src/utils/string_interner.rs b/shared/src/utils/string_interner.rs index d2431f432..968d68eb2 100644 --- a/shared/src/utils/string_interner.rs +++ b/shared/src/utils/string_interner.rs @@ -158,6 +158,19 @@ fn normalize_scalar_string(value: &str) -> &str { } } +/// Returns `true` for values that should be treated as absent. +/// +/// Covers: +/// - empty string (`""`) +/// - JSON/YAML null literals (`"null"`, `"~"`) +/// +/// Generic, field-agnostic. Callers decide whether `true` maps to `""` +/// (for non-optional `Arc`) or `None` (for `Option>`). +/// Case-sensitive on purpose: provider-supplied `"NULL"` or `"Null"` are real +/// values and must be preserved verbatim. +#[inline] +pub fn is_nullish(value: &str) -> bool { value.is_empty() || value == "~" || value.eq_ignore_ascii_case("null") } + // // Two reusable visitor types live here so that multiple public entry-points // can share them without code duplication: @@ -338,6 +351,142 @@ pub mod arc_str_option_null_if_empty_serde { pub use super::arc_str_option_serde::{deserialize, serialize_null_if_empty as serialize}; } +/// Visitor that mirrors `ArcStrVisitor` but collapses `"null"`, `""`, and `~` +/// to an empty `Arc`. Generic over the field, not specific to +/// `container_extension`. +struct NullishArcStrVisitor; + +impl<'de> Visitor<'de> for NullishArcStrVisitor { + type Value = Arc; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, boolean, or null (literal \"null\"/\"~\"/empty mapped to empty)") + } + + fn visit_str(self, v: &str) -> Result { + if is_nullish(v) { + Ok("".intern()) + } else { + Ok(normalize_scalar_string(v).intern()) + } + } + fn visit_string(self, v: String) -> Result { + if is_nullish(&v) { + Ok("".intern()) + } else { + Ok(normalize_scalar_string(&v).intern()) + } + } + fn visit_bool(self, v: bool) -> Result { Ok(v.to_string().intern()) } + fn visit_i64(self, v: i64) -> Result { Ok(v.to_string().intern()) } + fn visit_u64(self, v: u64) -> Result { Ok(v.to_string().intern()) } + fn visit_f64(self, v: f64) -> Result { Ok(f64_to_str(v).intern()) } + fn visit_unit(self) -> Result { Ok("".intern()) } + fn visit_none(self) -> Result { Ok("".intern()) } + fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } +} + +/// Visitor that mirrors `OptionArcStrVisitor` but collapses `"null"`, `""`, +/// and `~` to `None`. Generic over the field. +struct NullishOptionArcStrVisitor; + +impl<'de> Visitor<'de> for NullishOptionArcStrVisitor { + type Value = Option>; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, number, boolean, null, or empty (literal \"null\"/\"~\"/empty mapped to None)") + } + + fn visit_str(self, v: &str) -> Result { + if is_nullish(v) { + Ok(None) + } else { + Ok(Some(normalize_scalar_string(v).intern())) + } + } + fn visit_string(self, v: String) -> Result { + if is_nullish(&v) { + Ok(None) + } else { + Ok(Some(normalize_scalar_string(&v).intern())) + } + } + fn visit_bool(self, v: bool) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_i64(self, v: i64) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_u64(self, v: u64) -> Result { Ok(Some(v.to_string().intern())) } + fn visit_f64(self, v: f64) -> Result { Ok(Some(f64_to_str(v).intern())) } + fn visit_unit(self) -> Result { Ok(None) } + fn visit_none(self) -> Result { Ok(None) } + fn visit_some>(self, d: D) -> Result { d.deserialize_any(self) } + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + log::debug!("ignored sequence while deserializing nullish arc_str, returning None"); + Ok(None) + } + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + log::debug!("ignored map while deserializing nullish arc_str, returning None"); + Ok(None) + } +} + +/// Generic serde for `Arc` that treats JSON null, YAML null/`~`, empty +/// string, and the literal four-character string `"null"` as empty. +/// +/// Field-agnostic. Apply with `#[serde(with = "arc_str_null_is_none_serde")]`. +/// +/// Use instead of `arc_str_serde` whenever the underlying value might be +/// reported as `"null"` by a misbehaving provider. +pub mod arc_str_null_is_none_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::*; + + pub fn serialize(value: &Arc, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(value) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_option(NullishArcStrVisitor) + } +} + +/// Generic serde for `Option>` that treats JSON null, YAML null/`~`, +/// empty string, and the literal four-character string `"null"` as `None`. On +/// serialization, `None`, empty, and the literal `"null"` all become JSON null. +pub mod arc_str_null_is_none_option_serde { + // These serde helper modules deliberately re-use the parent module's imports. + // Clippy's expansion of this glob names private sibling modules and does not + // compile, so the glob is kept. + #[allow(clippy::wildcard_imports)] + use super::*; + + pub fn serialize(value: &Option>, serializer: S) -> Result + where + S: Serializer, + { + match value { + Some(s) if !super::is_nullish(s) => serializer.serialize_str(s), + _ => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_option(NullishOptionArcStrVisitor) + } +} + // // Reuses `ArcStrVisitor` / `OptionArcStrVisitor` via `deserialize_option`: // - null / ~ / empty -> visit_none / visit_unit -> "" / None @@ -445,4 +594,93 @@ mod tests { assert_eq!(parsed_neg_inf.value.as_deref(), Some("-infinity")); assert_eq!(parsed_nan.value.as_deref(), Some("nan")); } + + #[derive(Debug, serde::Deserialize, serde::Serialize)] + struct NullishArcStrHolder { + #[serde(default, with = "arc_str_null_is_none_serde")] + value: Arc, + } + + #[derive(Debug, serde::Deserialize, serde::Serialize)] + struct NullishOptArcStrHolder { + #[serde(default, with = "arc_str_null_is_none_option_serde")] + value: Option>, + } + + #[test] + fn null_is_none_arc_str_treats_literal_null_string_as_empty() { + let parsed: NullishArcStrHolder = serde_json::from_str(r#"{"value":"null"}"#).unwrap(); + assert!(parsed.value.is_empty()); + } + + #[test] + fn null_is_none_arc_str_treats_yaml_null_scalar_as_empty() { + let parsed: NullishArcStrHolder = serde_saphyr::from_str("value: null\n").unwrap(); + assert!(parsed.value.is_empty()); + let parsed_tilde: NullishArcStrHolder = serde_saphyr::from_str("value: ~\n").unwrap(); + assert!(parsed_tilde.value.is_empty()); + } + + #[test] + fn null_is_none_arc_str_treats_json_null_as_empty() { + let parsed: NullishArcStrHolder = serde_json::from_str(r#"{"value":null}"#).unwrap(); + assert!(parsed.value.is_empty()); + } + + #[test] + fn null_is_none_arc_str_preserves_real_extensions() { + for ext in ["mkv", "mp4", "ts", "avi"] { + let json = format!(r#"{{"value":"{ext}"}}"#); + let parsed: NullishArcStrHolder = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.value.as_ref(), ext, "extension {ext} must survive"); + } + } + + #[test] + fn null_is_none_option_treats_literal_null_string_as_none() { + let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":"null"}"#).unwrap(); + assert_eq!(parsed.value, None); + } + + #[test] + fn null_is_none_option_treats_json_null_as_none() { + let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":null}"#).unwrap(); + assert_eq!(parsed.value, None); + } + + #[test] + fn null_is_none_option_treats_empty_string_as_none() { + let parsed: NullishOptArcStrHolder = serde_json::from_str(r#"{"value":""}"#).unwrap(); + assert_eq!(parsed.value, None); + } + + #[test] + fn null_is_none_option_preserves_real_extensions() { + for ext in ["mkv", "mp4", "ts"] { + let json = format!(r#"{{"value":"{ext}"}}"#); + let parsed: NullishOptArcStrHolder = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.value.as_deref(), Some(ext), "extension {ext} must survive"); + } + } + + #[test] + fn null_is_none_option_serializes_none_and_literal_null_as_json_null() { + let none_value: NullishOptArcStrHolder = NullishOptArcStrHolder { value: None }; + assert_eq!(serde_json::to_string(&none_value).unwrap(), r#"{"value":null}"#); + + let literal_null: NullishOptArcStrHolder = NullishOptArcStrHolder { value: Some("null".into()) }; + assert_eq!(serde_json::to_string(&literal_null).unwrap(), r#"{"value":null}"#); + + let empty: NullishOptArcStrHolder = NullishOptArcStrHolder { value: Some("".into()) }; + assert_eq!(serde_json::to_string(&empty).unwrap(), r#"{"value":null}"#); + } + + #[test] + fn null_is_none_arc_str_roundtrips_real_extensions_byte_for_byte() { + let holder = NullishArcStrHolder { value: "mkv".into() }; + let json = serde_json::to_string(&holder).unwrap(); + assert_eq!(json, r#"{"value":"mkv"}"#); + let back: NullishArcStrHolder = serde_json::from_str(&json).unwrap(); + assert_eq!(back.value.as_ref(), "mkv"); + } }