From 0d4f113c54b2772fb782bbc1d861d40dab3afcb8 Mon Sep 17 00:00:00 2001 From: Juanjo Presa Date: Thu, 24 Sep 2026 17:36:34 +0200 Subject: [PATCH] feat: add TMDB Trending target curation (#872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add target-wide TMDB Trending discovery for movie/TV and day/week, bounded by an optional `limit` (default 100, range 1–500) of unique remote references before local matching - compose concrete Trakt and TMDB sources under one target curation policy while retaining legacy `output[].trakt` compatibility - match only positive TMDB IDs of the same media kind and preserve existing catalog identities, series closure, and Xtream alias behavior - require every active selector to complete before target IDs, writers, caches, or watches can change - preserve the YAML-owned block through Source Editor operations and add TMDB setup/failure documentation and application Credits attribution - validate complete decoded pages under independent request, selector, and shared target-batch budgets, using a configured TMDB-only HTTP profile with certificate/hostname verification and no redirects or replays --- CHANGELOG.md | 7 + Cargo.lock | 2 + README.md | 2 +- backend/app/src/api/api_utils/tests.rs | 9 + .../api/endpoints/custom_video_stream_api.rs | 1 + .../app/src/api/endpoints/v1_api_config.rs | 17 + .../app/src/api/endpoints/v1_api_playlist.rs | 6 + backend/app/src/api/endpoints/xmltv_api.rs | 2 + .../app/src/api/endpoints/xtream_api/tests.rs | 1 + backend/config-loader/src/lib.rs | 47 ++ .../src/runtime_config_report.rs | 12 + backend/core/src/model/config/app.rs | 1 + backend/core/src/model/config/curation.rs | 343 +++++++++ backend/core/src/model/config/mod.rs | 2 + backend/core/src/model/config/target.rs | 39 +- backend/core/src/model/xtream.rs | 1 + .../core/src/utils/network/content_coding.rs | 51 +- backend/core/src/utils/network/request.rs | 44 +- .../network/request/tests/tmdb_profile.rs | 185 +++++ backend/curation/Cargo.toml | 5 +- backend/curation/src/coordinator.rs | 110 +++ backend/curation/src/coordinator/tests.rs | 204 ++++++ backend/curation/src/lib.rs | 7 +- backend/curation/src/test_support.rs | 118 ++++ backend/curation/src/tmdb/client.rs | 257 +++++++ backend/curation/src/tmdb/client/tests.rs | 665 ++++++++++++++++++ backend/curation/src/tmdb/mod.rs | 100 +++ backend/curation/src/tmdb/model.rs | 146 ++++ backend/curation/src/trakt/mod.rs | 96 ++- backend/processing/Cargo.toml | 2 + .../processing/src/processor/playlist/mod.rs | 8 +- .../src/processor/playlist/target.rs | 85 ++- .../src/processor/playlist/tests.rs | 124 +++- .../playlist/tests/curation_publication.rs | 516 ++++++++++++++ .../client_diagnostics.rs | 108 +++ .../tests/curation_publication/item_limit.rs | 314 +++++++++ .../tests/curation_publication/tls_profile.rs | 106 +++ backend/repository/src/epg_repository.rs | 1 + backend/repository/src/playlist_repository.rs | 1 + backend/repository/src/xtream_repository.rs | 1 + config/source.yml | 25 + docs/src/configuration/config.md | 4 + docs/src/configuration/mapping-dsl.md | 4 +- docs/src/configuration/source.md | 205 +++++- frontend/public/assets/i18n/ar.json | 3 + frontend/public/assets/i18n/en.json | 3 + frontend/public/assets/i18n/ru.json | 3 + frontend/public/assets/tmdb-logo.NOTICE | 13 + frontend/public/assets/tmdb-logo.svg | 1 + frontend/scss/app/_component.scss | 1 + .../dashboard/_version_action_card.scss | 18 + .../dashboard/version_action_card.rs | 71 +- .../components/source_editor/editor_model.rs | 1 + .../components/source_editor/editor_view.rs | 33 + .../source_editor/output_xtream_form.rs | 29 +- .../components/source_editor/target_form.rs | 72 +- shared/src/model/config/curation.rs | 448 ++++++++++++ shared/src/model/config/mod.rs | 4 + shared/src/model/config/target.rs | 81 ++- shared/src/model/config/tmdb_curation.rs | 207 ++++++ shared/src/model/config/trakt.rs | 90 ++- 61 files changed, 4899 insertions(+), 163 deletions(-) create mode 100644 backend/core/src/model/config/curation.rs create mode 100644 backend/core/src/utils/network/request/tests/tmdb_profile.rs create mode 100644 backend/curation/src/coordinator.rs create mode 100644 backend/curation/src/coordinator/tests.rs create mode 100644 backend/curation/src/test_support.rs create mode 100644 backend/curation/src/tmdb/client.rs create mode 100644 backend/curation/src/tmdb/client/tests.rs create mode 100644 backend/curation/src/tmdb/mod.rs create mode 100644 backend/curation/src/tmdb/model.rs create mode 100644 backend/processing/src/processor/playlist/tests/curation_publication.rs create mode 100644 backend/processing/src/processor/playlist/tests/curation_publication/client_diagnostics.rs create mode 100644 backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs create mode 100644 backend/processing/src/processor/playlist/tests/curation_publication/tls_profile.rs create mode 100644 frontend/public/assets/tmdb-logo.NOTICE create mode 100644 frontend/public/assets/tmdb-logo.svg create mode 100644 frontend/scss/app/components/dashboard/_version_action_card.scss create mode 100644 shared/src/model/config/curation.rs create mode 100644 shared/src/model/config/tmdb_curation.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 673d9dad6..03faef5d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,6 +146,13 @@ ## 🌟 New Features +- **Target-owned discovery with TMDB Trending.** `target.curation` combines optional Trakt and TMDB sources under one + `full`/`curated` policy. TMDB supports movie/TV day/week feeds with explicit `scope: first_page`, its own Bearer token, + exact same-kind TMDB-ID matching, and optional Xtream category projection. Every active selector is required; failed + fetches retain finalized target artifacts in both modes. M3U/STRM keep normal selected entries, without Xtream aliases. + Legacy `output[].trakt` remains supported but cannot coexist with the new declaration on one target. Source Editor + preserves the YAML-owned block and shows ownership hints; the dashboard version card now includes TMDB Credits. + - **Runtime liveness watchdog.** An optional heartbeat/watchdog detects a wedged async runtime (process alive, scheduler no longer making progress, logs stop) and logs a diagnostic snapshot with runtime metrics and a per-thread `/proc/self/task` inventory. It is opt-in and off by default (`TULIPROX_WATCHDOG=1` diff --git a/Cargo.lock b/Cargo.lock index d41e1bec9..094ec2427 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4817,6 +4817,7 @@ dependencies = [ "path-clean", "quick-xml", "rand 0.9.5", + "rcgen", "regex", "reqwest", "rmp-serde", @@ -4828,6 +4829,7 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", "tokio-util", "tuliprox-core", "tuliprox-curation", diff --git a/README.md b/README.md index 4a19e37f5..2b3f97a2d 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Tuliprox can transform provider data into the structure you actually want to use - Output filters for the final playlist state - Favorites/bouquet management through `add_favourite(group_name)` - Target-specific bouquet whitelist/blacklist editing directly from the Web UI -- Trakt-based auto-curated bouquets such as trending and popular content +- Target-wide Trakt and TMDB Trending curation, with optional Xtream bouquets over locally available content **Benefit:** providers supply the raw catalog; Tuliprox decides what your users actually see and how it is organized. diff --git a/backend/app/src/api/api_utils/tests.rs b/backend/app/src/api/api_utils/tests.rs index b5a02de12..e1784971f 100644 --- a/backend/app/src/api/api_utils/tests.rs +++ b/backend/app/src/api/api_utils/tests.rs @@ -3448,6 +3448,7 @@ async fn dash_stream_request_remains_redirect_and_preserves_provider_affinity() ..ConfigInput::default() }); let target = Arc::new(ConfigTarget { + curation: None, id: 1, name: "target_dash".to_string(), enabled: true, @@ -5859,6 +5860,7 @@ async fn activate_session_before_stream_open_does_not_commit_user_lease_before_p fn create_test_shared_target() -> ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: "shared".to_string(), @@ -7464,6 +7466,7 @@ async fn local_stream_response_registers_active_local_stream() { let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; let user = ProxyUserCredentials::default(); let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), @@ -7523,6 +7526,7 @@ async fn local_stream_response_rechecks_limits_before_registering_socket_bound_s user.username = "local-limit-user".to_string(); user.max_connections = 1; let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), @@ -7848,6 +7852,7 @@ async fn stream_response_rolls_back_provisional_user_activation_when_provider_op let input_name = "provider_1".intern(); let input = app_state.app_config.get_input_by_name(&input_name).expect("provider input should exist"); let target = Arc::new(ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), @@ -8008,6 +8013,7 @@ async fn local_stream_response_disables_response_compression() { let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; let user = ProxyUserCredentials::default(); let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), @@ -8056,6 +8062,7 @@ async fn local_stream_response_reuses_stable_playback_session_token_across_reope let input = ConfigInput { input_type: InputType::Library, ..ConfigInput::default() }; let user = ProxyUserCredentials::default(); let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), @@ -8130,6 +8137,7 @@ async fn local_stream_response_allows_exhausted_reopen_for_same_playback_session user.username = "user1".to_string(); user.max_connections = 1; let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), @@ -8207,6 +8215,7 @@ async fn local_stream_response_preserves_soft_kind_across_reopens() { user.max_connections = 1; user.soft_connections = 1; let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "test".to_string(), diff --git a/backend/app/src/api/endpoints/custom_video_stream_api.rs b/backend/app/src/api/endpoints/custom_video_stream_api.rs index 066b3b990..56ba221a4 100644 --- a/backend/app/src/api/endpoints/custom_video_stream_api.rs +++ b/backend/app/src/api/endpoints/custom_video_stream_api.rs @@ -606,6 +606,7 @@ mod tests { ..ConfigInput::default() }); let target = Arc::new(ConfigTarget { + curation: None, id: 1, enabled: true, name: "target".to_string(), diff --git a/backend/app/src/api/endpoints/v1_api_config.rs b/backend/app/src/api/endpoints/v1_api_config.rs index a5d09b3ff..61b4fc029 100644 --- a/backend/app/src/api/endpoints/v1_api_config.rs +++ b/backend/app/src/api/endpoints/v1_api_config.rs @@ -1183,6 +1183,12 @@ mod tests { id: 1, enabled: false, name: String::from("Default"), + curation: Some( + serde_json::from_value( + serde_json::json!({"tmdb": {"api": {"access_token": "discovery-secret"}}}), + ) + .unwrap(), + ), output: vec![shared::model::TargetOutputDto::M3u(shared::model::M3uTargetOutputDto { filename: Some(String::from("secret.m3u")), ..Default::default() @@ -1221,6 +1227,7 @@ mod tests { assert_eq!(target.id, 1); assert_eq!(target.enabled, shared::model::ConfigTargetDto::default().enabled); assert!(target.output.is_empty()); + assert!(target.curation.is_none(), "identity-only access must not disclose discovery credentials"); assert!(target.filter.is_empty()); assert!(target.options.is_none()); assert!(target.sort.is_none()); @@ -1324,6 +1331,7 @@ mod tests { url: "http://example.com/playlist.m3u".to_string(), ..Default::default() }; + let curation: shared::model::CurationConfigDto = serde_json::from_value(serde_json::json!({"enabled": false, "catalog_selection": "curated", "tmdb": {"api": {"access_token": "discovery-test-token"}, "trending": [{"kind": "movie", "time_window": "week", "limit": 37, "create_xtream_category": false}]}})).unwrap(); let initial_sources = SourcesConfigDto { inputs: vec![input_dto.clone()], sources: vec![shared::model::ConfigSourceDto { @@ -1331,6 +1339,7 @@ mod tests { targets: vec![shared::model::ConfigTargetDto { id: 1, name: "target_1".to_string(), + curation: Some(curation.clone()), output: vec![shared::model::TargetOutputDto::M3u(shared::model::M3uTargetOutputDto::default())], ..Default::default() }], @@ -1382,6 +1391,7 @@ mod tests { targets: vec![shared::model::ConfigTargetDto { id: 1, name: "target_renamed".to_string(), + curation: Some(curation.clone()), output: vec![shared::model::TargetOutputDto::M3u(shared::model::M3uTargetOutputDto::default())], ..Default::default() }], @@ -1408,5 +1418,12 @@ mod tests { assert!(!tuliprox_repository::target_bouquet_exists(temp_dir.path(), "target_renamed").await); let runtime_sources = app_state.app_config.sources.load(); assert_eq!(runtime_sources.sources[0].targets[0].name, "target_renamed"); + assert_eq!( + shared::model::CurationConfigDto::from(runtime_sources.sources[0].targets[0].curation.as_ref().unwrap()), + curation + ); + let saved: SourcesConfigDto = + serde_saphyr::from_str(&tokio::fs::read_to_string(&sources_file).await.unwrap()).unwrap(); + assert_eq!(saved.sources[0].targets[0].curation.as_ref(), Some(&curation)); } } diff --git a/backend/app/src/api/endpoints/v1_api_playlist.rs b/backend/app/src/api/endpoints/v1_api_playlist.rs index a7e840c3f..dc5bacf45 100644 --- a/backend/app/src/api/endpoints/v1_api_playlist.rs +++ b/backend/app/src/api/endpoints/v1_api_playlist.rs @@ -1509,6 +1509,7 @@ mod tests { fn playlist_update_target(id: u16, name: &str) -> Arc { Arc::new(ConfigTarget { + curation: None, id, enabled: true, name: name.to_string(), @@ -2456,6 +2457,7 @@ mod tests { ConfigSource { inputs: vec!["input-a".intern()], targets: vec![Arc::new(ConfigTarget { + curation: None, id: 11, enabled: true, name: "stable-target".to_string(), @@ -2565,6 +2567,7 @@ mod tests { let input_b = Arc::new(ConfigInput { id: 8, name: "input-b".intern(), ..Default::default() }); let target = |id, name: &str| { Arc::new(ConfigTarget { + curation: None, id, enabled: true, name: name.to_string(), @@ -2801,6 +2804,7 @@ mod tests { execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), watch: None, use_memory_cache: false, + curation: None, }); let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![target] }; let app_config = test_app_config(input, source); @@ -2866,6 +2870,7 @@ mod tests { execution_plan: tuliprox_core::model::TargetExecutionPlan::default(), watch: None, use_memory_cache: false, + curation: None, }); let source = ConfigSource { inputs: vec![Arc::clone(&input.name)], targets: vec![target] }; let app_config = test_app_config(input, source); @@ -2902,6 +2907,7 @@ mod tests { ..Default::default() }); let target = Arc::new(ConfigTarget { + curation: None, id: 11, enabled: true, name: "target".to_string(), diff --git a/backend/app/src/api/endpoints/xmltv_api.rs b/backend/app/src/api/endpoints/xmltv_api.rs index f4a6e0d6f..a66458415 100644 --- a/backend/app/src/api/endpoints/xmltv_api.rs +++ b/backend/app/src/api/endpoints/xmltv_api.rs @@ -1056,6 +1056,7 @@ mod tests { fn test_target_with_xtream_and_m3u() -> ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: "mixed-target".to_string(), @@ -1088,6 +1089,7 @@ mod tests { fn test_target_with_xtream_only() -> ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: "xtream-only".to_string(), diff --git a/backend/app/src/api/endpoints/xtream_api/tests.rs b/backend/app/src/api/endpoints/xtream_api/tests.rs index bfc0b766c..d21240ec8 100644 --- a/backend/app/src/api/endpoints/xtream_api/tests.rs +++ b/backend/app/src/api/endpoints/xtream_api/tests.rs @@ -102,6 +102,7 @@ async fn response_body_text(response: axum::response::Response) -> Result ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: "ics-xtream".to_string(), diff --git a/backend/config-loader/src/lib.rs b/backend/config-loader/src/lib.rs index ad6ba02d6..6d814cd9a 100644 --- a/backend/config-loader/src/lib.rs +++ b/backend/config-loader/src/lib.rs @@ -1304,6 +1304,53 @@ mod tests { use tempfile::tempdir; use tuliprox_core::utils::resolve_env_var; + #[tokio::test] + async fn curation_item_limits_survive_source_load_sanitize_save_reload() { + let dir = tempdir().unwrap(); + let path = dir.path().join("source.yml"); + let document = r#" +inputs: [] +sources: + - inputs: [] + targets: + - name: discovery + enabled: false + output: + - type: xtream + curation: + enabled: false + tmdb: + enabled: false + api: + access_token: synthetic-test-token + trending: + - kind: movie + time_window: week + limit: 37 + create_xtream_category: false + category_name: Saved label + - kind: tv + time_window: day + limit: 100 + - kind: movie + time_window: day +"#; + tokio::fs::write(&path, document).await.unwrap(); + let dto = super::parse_sources_file_from_path(&path, false).await.unwrap(); + let expected = dto.sources[0].targets[0].curation.clone(); + let sanitized = sanitize_sources_for_persist(dto).await; + super::save_sources_config(path.to_str().unwrap(), dir.path().join("backup").to_str().unwrap(), &sanitized) + .await + .unwrap(); + let restored = super::parse_sources_file_from_path(&path, false).await.unwrap(); + assert_eq!(restored.sources[0].targets[0].curation, expected); + let yaml = tokio::fs::read_to_string(&path).await.unwrap(); + assert!(yaml.contains("limit: 37")); + assert!(!yaml.contains("limit: 100")); + tokio::fs::write(&path, document.replace("limit: 37", "scope: first_page")).await.unwrap(); + assert!(super::parse_sources_file_from_path(&path, false).await.is_err()); + } + #[test] #[allow(clippy::manual_unwrap_or_default)] fn test_resolve() { diff --git a/backend/config-loader/src/runtime_config_report.rs b/backend/config-loader/src/runtime_config_report.rs index a84cb8316..68b73141e 100644 --- a/backend/config-loader/src/runtime_config_report.rs +++ b/backend/config-loader/src/runtime_config_report.rs @@ -344,6 +344,18 @@ sources: assert_eq!(value["url"], "http://***/get.php?username=***&password=***"); } + #[test] + fn runtime_config_report_redacts_curation_bearer_token_in_target_arrays() { + let sources: shared::model::SourcesConfigDto = serde_json::from_value(serde_json::json!({ + "inputs": [], + "sources": [{"inputs": [], "targets": [{"name": "discovery", "curation": {"tmdb": {"api": {"access_token": "discovery-test-token"}}}}]}] + })).unwrap(); + let mut value = serde_json::json!({"sources": sources}); + redact_value(None, None, &mut value); + assert_eq!(value["sources"]["sources"][0]["targets"][0]["curation"]["tmdb"]["api"]["access_token"], "***"); + assert!(!value.to_string().contains("discovery-test-token")); + } + #[tokio::test] async fn runtime_config_report_formats_yaml() { let app_config = create_test_app_config().await; diff --git a/backend/core/src/model/config/app.rs b/backend/core/src/model/config/app.rs index a2a3f705f..2699a682f 100644 --- a/backend/core/src/model/config/app.rs +++ b/backend/core/src/model/config/app.rs @@ -617,6 +617,7 @@ mod tests { fn target_with_mapping_id(mapping_id: &str) -> Arc { Arc::new(ConfigTarget { + curation: None, id: 1, enabled: true, name: "target".to_string(), diff --git a/backend/core/src/model/config/curation.rs b/backend/core/src/model/config/curation.rs new file mode 100644 index 000000000..4fe2d5f32 --- /dev/null +++ b/backend/core/src/model/config/curation.rs @@ -0,0 +1,343 @@ +use crate::model::{macros, TraktApiConfig, TraktChartConfig, TraktConfig, TraktListConfig}; +use shared::model::{ + CurationCatalogSelection, CurationConfigDto, TmdbCurationApiConfigDto, TmdbCurationConfigDto, + TmdbTrendingConfigDto, TmdbTrendingKind, TmdbTrendingTimeWindow, TraktSourceConfigDto, +}; +use std::fmt; + +#[derive(Debug, Clone)] +pub struct TraktSourceConfig { + pub enabled: bool, + pub api: TraktApiConfig, + pub lists: Vec, + pub charts: Vec, +} + +macros::from_impl!(TraktSourceConfig); +impl From<&TraktSourceConfigDto> for TraktSourceConfig { + fn from(dto: &TraktSourceConfigDto) -> Self { + Self { + enabled: dto.enabled, + api: (&dto.api).into(), + lists: dto.lists.iter().map(Into::into).collect(), + charts: dto.charts.iter().map(Into::into).collect(), + } + } +} + +impl From<&TraktSourceConfig> for TraktSourceConfigDto { + fn from(config: &TraktSourceConfig) -> Self { + Self { + enabled: config.enabled, + api: (&config.api).into(), + lists: config.lists.iter().map(Into::into).collect(), + charts: config.charts.iter().map(Into::into).collect(), + } + } +} + +#[derive(Clone)] +pub struct TmdbCurationApiConfig { + pub access_token: String, +} + +impl fmt::Debug for TmdbCurationApiConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TmdbCurationApiConfig").field("access_token", &"[redacted]").finish() + } +} + +macros::from_impl!(TmdbCurationApiConfig); +impl From<&TmdbCurationApiConfigDto> for TmdbCurationApiConfig { + fn from(dto: &TmdbCurationApiConfigDto) -> Self { Self { access_token: dto.access_token.clone() } } +} + +impl From<&TmdbCurationApiConfig> for TmdbCurationApiConfigDto { + fn from(config: &TmdbCurationApiConfig) -> Self { Self { access_token: config.access_token.clone() } } +} + +#[derive(Debug, Clone)] +pub struct TmdbTrendingConfig { + pub kind: TmdbTrendingKind, + pub time_window: TmdbTrendingTimeWindow, + pub limit: u32, + pub category_name: Option, + pub create_xtream_category: bool, +} + +macros::from_impl!(TmdbTrendingConfig); +impl From<&TmdbTrendingConfigDto> for TmdbTrendingConfig { + fn from(dto: &TmdbTrendingConfigDto) -> Self { + Self { + kind: dto.kind, + time_window: dto.time_window, + limit: dto.limit, + category_name: dto.category_name.clone(), + create_xtream_category: dto.create_xtream_category, + } + } +} + +impl From<&TmdbTrendingConfig> for TmdbTrendingConfigDto { + fn from(config: &TmdbTrendingConfig) -> Self { + Self { + kind: config.kind, + time_window: config.time_window, + limit: config.limit, + category_name: config.category_name.clone(), + create_xtream_category: config.create_xtream_category, + } + } +} + +#[derive(Debug, Clone)] +pub struct TmdbCurationConfig { + pub enabled: bool, + pub api: TmdbCurationApiConfig, + pub trending: Vec, +} + +macros::from_impl!(TmdbCurationConfig); +impl From<&TmdbCurationConfigDto> for TmdbCurationConfig { + fn from(dto: &TmdbCurationConfigDto) -> Self { + Self { enabled: dto.enabled, api: (&dto.api).into(), trending: dto.trending.iter().map(Into::into).collect() } + } +} + +impl From<&TmdbCurationConfig> for TmdbCurationConfigDto { + fn from(config: &TmdbCurationConfig) -> Self { + Self { + enabled: config.enabled, + api: (&config.api).into(), + trending: config.trending.iter().map(Into::into).collect(), + } + } +} + +#[derive(Debug, Clone)] +pub struct CurationConfig { + pub enabled: bool, + pub catalog_selection: CurationCatalogSelection, + pub include_xtream_base_categories: bool, + pub trakt: Option, + pub tmdb: Option, +} + +macros::from_impl!(CurationConfig); +impl From<&CurationConfigDto> for CurationConfig { + fn from(dto: &CurationConfigDto) -> Self { + Self { + enabled: dto.enabled, + catalog_selection: dto.catalog_selection, + include_xtream_base_categories: dto.include_xtream_base_categories, + trakt: dto.trakt.as_ref().map(Into::into), + tmdb: dto.tmdb.as_ref().map(Into::into), + } + } +} + +impl From<&CurationConfig> for CurationConfigDto { + fn from(config: &CurationConfig) -> Self { + Self { + enabled: config.enabled, + catalog_selection: config.catalog_selection, + include_xtream_base_categories: config.include_xtream_base_categories, + trakt: config.trakt.as_ref().map(Into::into), + tmdb: config.tmdb.as_ref().map(Into::into), + } + } +} + +/// Execution-only normalization. Keep the original declaration when saving configuration. +impl From<&TraktConfig> for CurationConfig { + fn from(legacy: &TraktConfig) -> Self { + Self { + enabled: legacy.enabled, + catalog_selection: legacy.catalog_selection, + include_xtream_base_categories: legacy.include_xtream_base_categories, + trakt: Some(TraktSourceConfig { + enabled: true, + api: legacy.api.clone(), + lists: legacy.lists.clone(), + charts: legacy.charts.clone(), + }), + tmdb: None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use shared::model::{ + ConfigTargetDto, TargetOutputDto, TraktApiConfigDto, TraktChartConfigDto, TraktChartKind, TraktChartType, + TraktConfigDto, TraktContentType, TraktListConfigDto, XtreamTargetOutputDto, + }; + + fn legacy_config() -> TraktConfigDto { + TraktConfigDto { + api: TraktApiConfigDto { + api_key: "client-id".to_string(), + version: "2".to_string(), + url: "https://api.trakt.tv".to_string(), + user_agent: "custom-agent".to_string(), + }, + lists: vec![TraktListConfigDto { + user: "alice".to_string(), + list_slug: "watchlist".to_string(), + content_type: TraktContentType::Both, + category_name: Some("Saved list label".to_string()), + create_xtream_category: false, + tmdb_only: false, + fuzzy_match_threshold: 83, + }], + charts: vec![TraktChartConfigDto { + kind: TraktChartKind::Shows, + chart: TraktChartType::Popular, + category_name: Some("Shows".to_string()), + create_xtream_category: true, + tmdb_only: true, + fuzzy_match_threshold: 91, + }], + ..TraktConfigDto::default() + } + } + + #[test] + fn curation_legacy_normalization_preserves_all_policy_axes_and_source_settings() { + for enabled in [false, true] { + for selection in [CurationCatalogSelection::Full, CurationCatalogSelection::Curated] { + for include_base in [false, true] { + let mut legacy = legacy_config(); + legacy.enabled = enabled; + legacy.catalog_selection = selection; + legacy.include_xtream_base_categories = include_base; + let mut canonical = CurationConfigDto { + enabled, + catalog_selection: selection, + include_xtream_base_categories: include_base, + trakt: Some(TraktSourceConfigDto { + enabled: true, + api: legacy.api.clone(), + lists: legacy.lists.clone(), + charts: legacy.charts.clone(), + }), + tmdb: None, + }; + legacy.prepare().unwrap(); + canonical.prepare(&[TargetOutputDto::Xtream(XtreamTargetOutputDto::default())]).unwrap(); + let resolved_legacy = TraktConfig::from(&legacy); + let normalized = CurationConfig::from(&resolved_legacy); + assert_eq!(CurationConfigDto::from(&normalized), canonical); + assert_eq!( + TraktConfigDto::from(&resolved_legacy), + legacy, + "normalization does not rewrite ownership" + ); + } + } + } + } + + #[test] + fn curation_legacy_source_less_disabled_policy_is_not_lost_on_normalization() { + for policy in [ + TraktConfigDto { + enabled: false, + catalog_selection: CurationCatalogSelection::Curated, + ..TraktConfigDto::default() + }, + TraktConfigDto { enabled: false, include_xtream_base_categories: false, ..TraktConfigDto::default() }, + TraktConfigDto::default(), + ] { + let normalized = CurationConfig::from(&TraktConfig::from(&policy)); + assert_eq!(normalized.enabled, policy.enabled); + assert_eq!(normalized.catalog_selection, policy.catalog_selection); + assert_eq!(normalized.include_xtream_base_categories, policy.include_xtream_base_categories); + let source = normalized.trakt.unwrap(); + assert!(source.lists.is_empty()); + assert!(source.charts.is_empty()); + assert!(source.enabled, "legacy whole-block switch belongs to the target policy"); + } + } + + #[test] + fn curation_round_trip_preserves_source_switches_credentials_and_selection_only_labels() { + let value = serde_json::json!({ + "enabled": false, + "catalog_selection": "curated", + "include_xtream_base_categories": false, + "trakt": { + "enabled": false, + "api": {"api_key": "trakt-client", "url": "https://api.trakt.tv"}, + "charts": [{"kind": "movies", "chart": "trending", "category_name": "Trakt", "tmdb_only": true}] + }, + "tmdb": { + "api": {"access_token": "private-test-token"}, + "trending": [ + {"kind": "movie", "time_window": "day", "limit": 1, "category_name": "Saved label", "create_xtream_category": false}, + {"kind": "tv", "time_window": "week", "limit": 500, "category_name": "Shows"} + ] + } + }); + let dto: CurationConfigDto = serde_json::from_value(value).unwrap(); + let resolved = CurationConfig::from(&dto); + assert_eq!(CurationConfigDto::from(&resolved), dto); + assert!(!format!("{resolved:?}").contains("private-test-token")); + assert!(!format!("{dto:?}").contains("private-test-token")); + } + + #[test] + fn curation_item_limit_json_yaml_prepare_runtime_round_trip() { + for limit in [None, Some(100), Some(1), Some(500)] { + let mut selector = serde_json::json!({"kind": "movie", "time_window": "day", "category_name": "Movies"}); + if let Some(limit) = limit { + selector["limit"] = limit.into(); + } + let value = serde_json::json!({"tmdb": {"trending": [selector]}}); + let dto: CurationConfigDto = serde_json::from_value(value).unwrap(); + let yaml = serde_saphyr::to_string(&dto).unwrap(); + let mut dto: CurationConfigDto = serde_saphyr::from_str(&yaml).unwrap(); + dto.prepare(&[TargetOutputDto::Xtream(XtreamTargetOutputDto::default())]).unwrap(); + let runtime = CurationConfig::from(&dto); + assert_eq!(runtime.tmdb.as_ref().unwrap().trending[0].limit, limit.unwrap_or(100)); + let restored = CurationConfigDto::from(&runtime); + assert_eq!(restored, dto); + let serialized = serde_json::to_value(&restored).unwrap(); + assert_eq!(serialized["tmdb"]["trending"][0].get("limit").is_none(), limit.unwrap_or(100) == 100); + assert_eq!( + serde_saphyr::from_str::(&serde_saphyr::to_string(&restored).unwrap()).unwrap(), + dto + ); + } + } + + #[test] + fn curation_round_trip_preserves_absent_sources_and_empty_blocks() { + for value in + [serde_json::json!({}), serde_json::json!({"trakt": {}}), serde_json::json!({"tmdb": {"enabled": false}})] + { + let dto: CurationConfigDto = serde_json::from_value(value).unwrap(); + assert_eq!(CurationConfigDto::from(&CurationConfig::from(&dto)), dto); + } + } + + #[test] + fn curation_legacy_target_output_round_trip_keeps_its_original_declaration() { + let mut dto = ConfigTargetDto { + name: "discovery".to_string(), + output: vec![TargetOutputDto::Xtream(XtreamTargetOutputDto { + trakt: Some(legacy_config()), + ..XtreamTargetOutputDto::default() + })], + ..ConfigTargetDto::default() + }; + dto.prepare(1, None, None).unwrap(); + let resolved = crate::model::ConfigTarget::from(&dto); + let outputs: Vec = resolved.output.iter().map(Into::into).collect(); + assert_eq!(outputs, dto.output); + let serialized = serde_json::to_value(&outputs).unwrap(); + assert!(serialized[0].get("trakt").is_some()); + assert!(serialized[0].get("curation").is_none()); + } +} diff --git a/backend/core/src/model/config/mod.rs b/backend/core/src/model/config/mod.rs index cf66b8e1c..57900f606 100644 --- a/backend/core/src/model/config/mod.rs +++ b/backend/core/src/model/config/mod.rs @@ -4,6 +4,7 @@ mod api_user; mod app; mod base; mod cache; +mod curation; mod epg; mod epg_smart_match; mod favourites; @@ -41,6 +42,7 @@ pub use api_proxy::*; pub use api_user::*; pub use app::*; pub use base::*; +pub use curation::*; pub use epg::*; pub use epg_smart_match::*; pub use favourites::*; diff --git a/backend/core/src/model/config/target.rs b/backend/core/src/model/config/target.rs index dd95ba777..f900e0af5 100644 --- a/backend/core/src/model/config/target.rs +++ b/backend/core/src/model/config/target.rs @@ -2,7 +2,7 @@ use crate::model::{ config::{favourites::ConfigFavourites, trakt::TraktConfig}, macros, mapping::CompiledMapping, - ConfigRename, ConfigSort, + ConfigRename, ConfigSort, CurationConfig, }; use arc_swap::ArcSwapOption; use shared::{ @@ -14,7 +14,7 @@ use shared::{ TargetType, TraktConfigDto, XtreamTargetOutputDto, }, }; -use std::sync::Arc; +use std::{borrow::Cow, sync::Arc}; create_bitset!(u8, XtreamTargetFlags, SkipLiveDirectSource, SkipVideoDirectSource, SkipSeriesDirectSource); create_bitset!(u8, StrmTargetFlags, Flat, UnderscoreWhitespace, Cleanup, AddQualityToFilename, UseMetadata); @@ -328,6 +328,7 @@ pub struct ConfigTarget { pub sort: Option, pub filter: StagedFilter, pub output: Vec, + pub curation: Option, pub rename: Option>, pub mapping_ids: Option>, pub mapping: Arc>, @@ -339,6 +340,15 @@ pub struct ConfigTarget { } impl ConfigTarget { + /// Normalize only for execution; the original config declaration remains untouched. + pub fn effective_curation(&self) -> Option> { + self.curation.as_ref().map(Cow::Borrowed).or_else(|| { + self.get_xtream_output() + .and_then(|output| output.trakt.as_ref()) + .map(|legacy| Cow::Owned(CurationConfig::from(legacy))) + }) + } + pub fn filter(&self, provider: &ValueProvider) -> bool { self.filter.processing.as_ref().is_none_or(|filter| filter.filter(provider)) } @@ -388,6 +398,7 @@ impl From<&ConfigTargetDto> for ConfigTarget { sort: dto.sort.as_ref().map(Into::into), filter: StagedFilter { processing: dto.filter.t_processing.clone(), persist: dto.filter.t_persist.clone() }, output: dto.output.iter().map(Into::into).collect(), + curation: dto.curation.as_ref().map(Into::into), rename: dto.rename.as_ref().map(|l| l.iter().map(Into::into).collect()), mapping_ids: dto.mapping.clone(), mapping: Arc::new(ArcSwapOption::new(None)), @@ -421,6 +432,30 @@ impl From<&ConfigTargetDto> for ConfigTarget { mod tests { use super::*; + #[test] + fn effective_curation_retains_canonical_owner_and_credentials_without_legacy_output() { + let mut dto: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"api": {"access_token": "test-token"}, "trending": [{"kind": "tv", "time_window": "day", "limit": 37, "create_xtream_category": false}]}}})).unwrap(); + dto.prepare(1, None, None).unwrap(); + let target = ConfigTarget::from(&dto); + assert!(matches!(target.effective_curation(), Some(Cow::Borrowed(_)))); + assert_eq!(shared::model::CurationConfigDto::from(target.curation.as_ref().unwrap()), dto.curation.unwrap()); + assert!(target.get_xtream_output().is_none()); + assert!(!format!("{target:?}").contains("test-token")); + } + + #[test] + fn effective_curation_normalizes_legacy_only_for_execution() { + let mut dto: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "legacy", "output": [{"type": "xtream", "trakt": {"enabled": false, "catalog_selection": "curated"}}]})).unwrap(); + dto.prepare(1, None, None).unwrap(); + let target = ConfigTarget::from(&dto); + assert!(matches!(target.effective_curation(), Some(Cow::Owned(_)))); + assert!(target.curation.is_none()); + assert!(!target.effective_curation().unwrap().enabled); + assert!(target.effective_curation().unwrap().catalog_selection.is_curated()); + assert_eq!(TargetOutputDto::from(&target.output[0]), dto.output[0]); + } + #[test] fn execution_plan_compiles_processing_order() { let dto = ConfigTargetDto { processing_order: ProcessingOrder::Mrf, ..Default::default() }; diff --git a/backend/core/src/model/xtream.rs b/backend/core/src/model/xtream.rs index 70974c8da..216cb470e 100644 --- a/backend/core/src/model/xtream.rs +++ b/backend/core/src/model/xtream.rs @@ -184,6 +184,7 @@ mod tests { fn create_test_target() -> (ConfigTarget, XtreamTargetOutput) { let xtream_output = XtreamTargetOutput { flags: XtreamTargetFlagsSet::default(), trakt: None, filter: None }; let target = ConfigTarget { + curation: None, id: 1, enabled: true, name: "xtream-target".to_string(), diff --git a/backend/core/src/utils/network/content_coding.rs b/backend/core/src/utils/network/content_coding.rs index cd0df981f..aca047ca9 100644 --- a/backend/core/src/utils/network/content_coding.rs +++ b/backend/core/src/utils/network/content_coding.rs @@ -224,6 +224,23 @@ pub fn parse_content_codings(headers: &HeaderMap) -> Result, pub async fn decode_response_to_identity( response: reqwest::Response, detection: ContentCodingDetection, +) -> Result { + decode_response(response, detection, false).await +} + +/// Opt-in complete-body consumer: decode all members and require underlying HTTP +/// EOF. A finished compressed member must not hide a truncated transport or suffix. +/// Existing streaming/text consumers retain their current decoding policy. +pub async fn decode_complete_response_to_identity( + response: reqwest::Response, +) -> Result { + decode_response(response, ContentCodingDetection::DeclaredOnly, true).await +} + +async fn decode_response( + response: reqwest::Response, + detection: ContentCodingDetection, + complete: bool, ) -> Result { let status = response.status(); let final_url = response.url().clone(); @@ -257,7 +274,7 @@ pub async fn decode_response_to_identity( } for coding in decoded_from.iter().rev().copied() { - body = decoder_for(body, coding).await?; + body = decoder_for(body, coding, complete).await?; } let mut decoded = DecodedHttpResponse { status, final_url, headers, body, decoded_from, original_content_length }; @@ -330,18 +347,38 @@ pub fn content_decoding_error_from_io(error: &io::Error) -> Option<&ContentDecod find_content_decoding_error(error.get_ref()?) } -async fn decoder_for(mut reader: DynReader, coding: ContentCoding) -> Result { +async fn decoder_for( + mut reader: DynReader, + coding: ContentCoding, + complete: bool, +) -> Result { let reader = match coding { - ContentCoding::Gzip => tagged_decoder(GzipDecoder::new(BufReader::new(reader)), coding), - ContentCoding::Brotli => tagged_decoder(BrotliDecoder::new(BufReader::new(reader)), coding), - ContentCoding::Zstd => tagged_decoder(ZstdDecoder::new(BufReader::new(reader)), coding), + ContentCoding::Gzip => { + let mut decoder = GzipDecoder::new(BufReader::new(reader)); + decoder.multiple_members(complete); + tagged_decoder(decoder, coding) + } + ContentCoding::Brotli => { + let mut decoder = BrotliDecoder::new(BufReader::new(reader)); + decoder.multiple_members(complete); + tagged_decoder(decoder, coding) + } + ContentCoding::Zstd => { + let mut decoder = ZstdDecoder::new(BufReader::new(reader)); + decoder.multiple_members(complete); + tagged_decoder(decoder, coding) + } ContentCoding::Deflate => { let (replayed_reader, prefix) = inspect_prefix(reader, 2).await?; reader = replayed_reader; if is_zlib_header(&prefix) { - tagged_decoder(ZlibDecoder::new(BufReader::new(reader)), coding) + let mut decoder = ZlibDecoder::new(BufReader::new(reader)); + decoder.multiple_members(complete); + tagged_decoder(decoder, coding) } else { - tagged_decoder(DeflateDecoder::new(BufReader::new(reader)), coding) + let mut decoder = DeflateDecoder::new(BufReader::new(reader)); + decoder.multiple_members(complete); + tagged_decoder(decoder, coding) } } }; diff --git a/backend/core/src/utils/network/request.rs b/backend/core/src/utils/network/request.rs index ec0a293d8..b7d964e33 100644 --- a/backend/core/src/utils/network/request.rs +++ b/backend/core/src/utils/network/request.rs @@ -3007,8 +3007,11 @@ pub async fn get_input_json_content_as_stream( } pub fn create_client_with_redirect(cfg: &AppConfig, redirect_policy: Policy) -> reqwest::ClientBuilder { - let config = cfg.config.load(); - log_proxy_diagnostics(&config); + configured_client(&cfg.config.load(), redirect_policy) +} + +fn configured_client(config: &Config, redirect_policy: Policy) -> reqwest::ClientBuilder { + log_proxy_diagnostics(config); let mut client = reqwest::Client::builder() .redirect(redirect_policy) .pool_idle_timeout(Duration::from_secs(30)) @@ -3069,6 +3072,41 @@ pub fn create_client(cfg: &AppConfig) -> reqwest::ClientBuilder { create_client_with_redirect(cfg, Policy::limited(10)) } +/// Dedicated discovery profile. Callers must propagate construction failure, never +/// fall back to an unconfigured client. Builder access permits local CA/DNS fixtures. +pub fn create_tmdb_client(cfg: &AppConfig) -> Result { + let config = cfg.config.load(); + let invalid_proxy = || TuliproxError::Config("TMDB HTTP profile has invalid proxy configuration".to_string()); + // The general factory historically logs malformed proxies and continues. TMDB + // must fail closed, using the same snapshot and proxy transformations instead. + if let Some(proxy) = &config.proxy { + let mut url = Url::parse(&proxy.url).map_err(|_| invalid_proxy())?; + match url.scheme() { + "socks5" | "socks5h" => { + if let Some(user) = &proxy.username { + url.set_username(user).map_err(|()| invalid_proxy())?; + } + if let Some(pass) = &proxy.password { + url.set_password(Some(pass)).map_err(|()| invalid_proxy())?; + } + } + "http" | "https" => {} + _ => return Err(invalid_proxy()), + } + reqwest::Proxy::all(url.as_str()).map_err(|_| invalid_proxy())?; + } + // Discovery carries its own Bearer credential: never inherit the generic TLS bypass. + // Keep configured proxy/auth and trusted roots, including hostname verification. + let mut builder = configured_client(&config, Policy::none()) + .danger_accept_invalid_certs(false) + .retry(reqwest::retry::never()) + .http1_only(); + if config.connect_timeout_secs > 0 { + builder = builder.connect_timeout(Duration::from_secs(u64::from(config.connect_timeout_secs))); + } + Ok(builder) +} + /// Redirect hops allowed for a resource fetch. The same bound the general client uses. pub const RESOURCE_REDIRECT_LIMIT: usize = 10; @@ -3204,6 +3242,8 @@ pub fn should_trigger_failover(status: StatusCode) -> bool { #[cfg(test)] mod tests { + mod tmdb_profile; + use super::{ classify_ip, download_text_content, download_text_content_with_headers_and_options, get_input_epg_content_as_file, get_remote_content_as_stream, is_safe_cross_origin_redirect_header, diff --git a/backend/core/src/utils/network/request/tests/tmdb_profile.rs b/backend/core/src/utils/network/request/tests/tmdb_profile.rs new file mode 100644 index 000000000..6142c88c1 --- /dev/null +++ b/backend/core/src/utils/network/request/tests/tmdb_profile.rs @@ -0,0 +1,185 @@ +//! Physical traffic tests for the production configured discovery profile. +use super::{make_test_app_config, response_with_body, start_hanging_http_server, start_recording_http_server}; +use crate::{ + model::{Config, ProxyConfig}, + utils::network::request::{create_client, create_tmdb_client}, +}; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; + +#[tokio::test] +async fn tmdb_profile_redirects_send_only_one_get_and_never_contact_a_second_destination() { + let (foreign, foreign_requests, foreign_task) = + start_recording_http_server(vec![response_with_body("200 OK", "ok")]).await.unwrap(); + let cfg = make_test_app_config(Config::default()); + for location in [ + format!("http://{foreign}/foreign"), + "/same-origin".into(), + "http://api.themoviedb.org/downgrade".into(), + "/start".into(), + ] { + let (address, requests, task) = start_recording_http_server(vec![format!( + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )]) + .await + .unwrap(); + let client = create_tmdb_client(&cfg).unwrap().no_proxy().build().unwrap(); + let response = + client.get(format!("http://{address}/start")).header(AUTHORIZATION, "Bearer fixture").send().await.unwrap(); + assert_eq!(response.status(), 302); + assert_eq!(requests.lock().await.len(), 1); + assert!(foreign_requests.lock().await.is_empty()); + task.abort(); + } + foreign_task.abort(); +} + +#[tokio::test] +async fn tmdb_profile_retains_authenticated_proxy_and_default_headers() { + let (proxy, requests, task) = + start_recording_http_server(vec![response_with_body("200 OK", "proxied")]).await.unwrap(); + let cfg = make_test_app_config(Config { + proxy: Some(ProxyConfig { + url: format!("http://{proxy}"), + username: Some("proxy-user".into()), + password: Some("proxy-password".into()), + }), + connect_timeout_secs: 1, + ..Config::default() + }); + let mut headers = HeaderMap::new(); + headers.insert("x-context", HeaderValue::from_static("retained")); + let client = create_tmdb_client(&cfg).unwrap().default_headers(headers).build().unwrap(); + let response = client + .get("http://unresolvable.invalid/trending") + .header(AUTHORIZATION, "Bearer discovery-token") + .send() + .await + .unwrap(); + assert_eq!(response.text().await.unwrap(), "proxied"); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + let request = requests[0].to_lowercase(); + assert!(request.starts_with("get http://unresolvable.invalid/trending http/1.1")); + assert!(request.contains("proxy-authorization: basic ")); + assert!(request.contains("authorization: bearer discovery-token")); + assert!(request.contains("x-context: retained")); + task.abort(); +} + +#[tokio::test] +async fn tmdb_profile_connect_timeout_covers_a_hanging_proxy_tunnel() { + let (proxy, seen, task) = start_hanging_http_server().await.unwrap(); + let cfg = make_test_app_config(Config { + proxy: Some(ProxyConfig { url: format!("http://{proxy}"), username: None, password: None }), + connect_timeout_secs: 1, + ..Config::default() + }); + let client = create_tmdb_client(&cfg).unwrap().build().unwrap(); + let result = + tokio::time::timeout(Duration::from_secs(3), client.get("https://unresolvable.invalid/trending").send()) + .await + .unwrap(); + seen.await.unwrap(); + assert!(result.unwrap_err().is_timeout()); + task.abort(); +} + +#[test] +fn tmdb_profile_construction_errors_are_not_unconfigured_fallbacks() { + for proxy in ["not a url", "file:///tmp/proxy", "http://[invalid"] { + let cfg = make_test_app_config(Config { + proxy: Some(ProxyConfig { url: proxy.into(), username: None, password: None }), + ..Config::default() + }); + let error = create_tmdb_client(&cfg).err().expect("malformed proxy must fail closed"); + assert!(!error.to_string().contains(proxy)); + } + let cfg = make_test_app_config(Config::default()); + assert!(create_tmdb_client(&cfg).unwrap().user_agent("bad\nheader").build().is_err()); +} + +#[tokio::test] +async fn tmdb_profile_refused_http2_stream_is_not_replayed_unlike_the_generic_default() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let gets = Arc::new(AtomicUsize::new(0)); + let recorded = Arc::clone(&gets); + let task = tokio::spawn(async move { + let mut connections = tokio::task::JoinSet::new(); + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let gets = Arc::clone(&recorded); + connections.spawn(async move { + let mut preface = [0; 24]; + stream.read_exact(&mut preface).await.unwrap(); + assert_eq!(&preface, b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"); + // Empty SETTINGS. Each HEADERS is an actually received request; + // REFUSED_STREAM (7) is reqwest's default protocol retry trigger. + stream.write_all(&[0, 0, 0, 4, 0, 0, 0, 0, 0]).await.unwrap(); + loop { + let mut header = [0; 9]; + if stream.read_exact(&mut header).await.is_err() { + break; + } + let length = + (usize::from(header[0]) << 16) | (usize::from(header[1]) << 8) | usize::from(header[2]); + assert!(length <= 65536); + let mut payload = vec![0; length]; + stream.read_exact(&mut payload).await.unwrap(); + if header[3] == 4 && header[4] == 0 { + stream.write_all(&[0, 0, 0, 4, 1, 0, 0, 0, 0]).await.unwrap(); + } else if header[3] == 1 { + gets.fetch_add(1, Ordering::SeqCst); + let mut reset = vec![0, 0, 4, 3, 0]; + reset.extend_from_slice(&header[5..9]); + reset.extend_from_slice(&[0, 0, 0, 7]); + stream.write_all(&reset).await.unwrap(); + } + } + }); + } + }); + let cfg = make_test_app_config(Config::default()); + for (builder, expected) in [(create_tmdb_client(&cfg).unwrap(), 1), (create_client(&cfg), 3)] { + let before = gets.load(Ordering::SeqCst); + let client = builder.no_proxy().http2_prior_knowledge().timeout(Duration::from_secs(2)).build().unwrap(); + assert!(client.get(format!("http://{address}/trending")).send().await.is_err()); + assert_eq!(gets.load(Ordering::SeqCst) - before, expected, "count actual HEADERS, not send calls"); + } + task.abort(); +} + +#[tokio::test] +async fn tmdb_profile_reuses_or_recovers_a_closed_idle_connection_without_replaying_sent_gets() { + let (address, requests, task) = start_recording_http_server(vec![ + // Advertise keepalive but close after replying: the second logical request + // may recover an unused idle connection, never replay a GET already sent. + "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok".into(), + ]) + .await + .unwrap(); + let cfg = make_test_app_config(Config::default()); + let client = create_tmdb_client(&cfg).unwrap().no_proxy().build().unwrap(); + for _ in 0..2 { + let response = + tokio::time::timeout(Duration::from_secs(1), client.get(format!("http://{address}/trending")).send()) + .await + .unwrap() + .unwrap(); + assert_eq!(response.text().await.unwrap(), "ok"); + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(requests.lock().await.len(), 2); + task.abort(); +} diff --git a/backend/curation/Cargo.toml b/backend/curation/Cargo.toml index e2c3d0bcb..be3280d86 100644 --- a/backend/curation/Cargo.toml +++ b/backend/curation/Cargo.toml @@ -3,7 +3,7 @@ name = "tuliprox-curation" version.workspace = true edition.workspace = true rust-version.workspace = true -description = "Source-neutral playlist curation with a Trakt edge adapter for tuliprox" +description = "Source-neutral playlist curation with Trakt and TMDB Trending adapters for tuliprox" [lints] workspace = true @@ -19,6 +19,7 @@ reqwest = { version = "0.13.4", features = ["json", "rustls"] } serde.workspace = true serde_json.workspace = true strsim = "0.11.1" +tokio = { workspace = true, features = ["io-util", "time"] } [dev-dependencies] -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "test-util"] } diff --git a/backend/curation/src/coordinator.rs b/backend/curation/src/coordinator.rs new file mode 100644 index 000000000..4ed3dbbd8 --- /dev/null +++ b/backend/curation/src/coordinator.rs @@ -0,0 +1,110 @@ +use crate::{ + kernel::{ + CurationEvaluation, CurationFailure, CurationProjectionCatalog, CurationRunOutcome, CurationSelectorKey, + CurationSelectorSummary, SelectorOutcome, + }, + tmdb::{self, TmdbClient, TmdbFailure}, + trakt, +}; +use shared::model::PlaylistGroup; +use tuliprox_core::model::CurationConfig; + +#[cfg(test)] +mod tests; + +// Persistence compatibility at the projection edge, not the identity of a provider. +pub(crate) const LEGACY_CATEGORY_NAMESPACE: &str = "trakt-category"; + +fn selector_keys(config: &CurationConfig) -> (Vec, Vec) { + if !config.enabled { + return (Vec::new(), Vec::new()); + } + let trakt_count = config + .trakt + .as_ref() + .filter(|source| source.enabled) + .map_or(0, |source| source.lists.len() + source.charts.len()); + let tmdb_count = config.tmdb.as_ref().filter(|source| source.enabled).map_or(0, |source| source.trending.len()); + ( + (0..trakt_count).map(CurationSelectorKey).collect(), + (trakt_count..trakt_count + tmdb_count).map(CurationSelectorKey).collect(), + ) +} + +/// Evaluate all required selectors against the same eligible, pre-projection catalog. +pub async fn evaluate_curation( + http: &reqwest::Client, + tmdb_http: Option<&reqwest::Client>, + playlist: &[PlaylistGroup], + target: &str, + config: &CurationConfig, +) -> CurationRunOutcome { + let tmdb_client = + config.tmdb.as_ref().filter(|source| config.enabled && source.enabled && !source.trending.is_empty()).map( + |source| tmdb_http.ok_or(TmdbFailure::Configuration).and_then(|http| TmdbClient::new(http, &source.api)), + ); + evaluate_with_tmdb(http, playlist, target, config, tmdb_client).await +} + +async fn evaluate_with_tmdb( + http: &reqwest::Client, + playlist: &[PlaylistGroup], + target: &str, + config: &CurationConfig, + tmdb_client: Option>, +) -> CurationRunOutcome { + let (trakt_keys, tmdb_keys) = selector_keys(config); + let mut outcomes = Vec::with_capacity(trakt_keys.len() + tmdb_keys.len()); + if !trakt_keys.is_empty() { + if let Some(source) = &config.trakt { + outcomes.extend(trakt::evaluate_selectors(http, playlist, target, source, &trakt_keys).await); + } + } + if !tmdb_keys.is_empty() { + if let Some(source) = &config.tmdb { + let client = tmdb_client.unwrap_or(Err(TmdbFailure::Configuration)); + outcomes.extend(tmdb::evaluate_selectors(client, playlist, target, source, &tmdb_keys).await); + } + } + complete_evaluation(outcomes) +} + +pub fn project_curation_categories( + evaluation: &CurationEvaluation, + playlist: &[PlaylistGroup], + config: &CurationConfig, +) -> Vec { + let (trakt_keys, tmdb_keys) = selector_keys(config); + let catalog = CurationProjectionCatalog::new(playlist); + let mut groups = Vec::new(); + if !trakt_keys.is_empty() { + if let Some(source) = &config.trakt { + groups.extend(trakt::project_categories(evaluation, &catalog, source, &trakt_keys)); + } + } + if !tmdb_keys.is_empty() { + if let Some(source) = &config.tmdb { + groups.extend(tmdb::project_categories(evaluation, &catalog, source, &tmdb_keys)); + } + } + groups +} + +pub(crate) fn complete_evaluation(selector_outcomes: Vec) -> CurationRunOutcome { + if selector_outcomes.is_empty() { + return CurationRunOutcome::NotConfigured; + } + if selector_outcomes.iter().any(|outcome| !matches!(outcome, SelectorOutcome::Complete { .. })) { + return CurationRunOutcome::Failed(CurationFailure { selector_outcomes }); + } + let mut selectors = Vec::with_capacity(selector_outcomes.len()); + let mut memberships = Vec::new(); + for outcome in selector_outcomes { + let SelectorOutcome::Complete { key, reference_count, memberships: mut selector_memberships } = outcome else { + unreachable!("all selector outcomes were checked as complete") + }; + selectors.push(CurationSelectorSummary { key, reference_count, membership_count: selector_memberships.len() }); + memberships.append(&mut selector_memberships); + } + CurationRunOutcome::Complete(CurationEvaluation { selectors, memberships }) +} diff --git a/backend/curation/src/coordinator/tests.rs b/backend/curation/src/coordinator/tests.rs new file mode 100644 index 000000000..c0ef275d4 --- /dev/null +++ b/backend/curation/src/coordinator/tests.rs @@ -0,0 +1,204 @@ +use super::*; +use crate::{ + kernel::{CurationMediaKind, CurationUnavailableReason}, + test_support::{http_response, TestServer}, +}; +use serde_json::json; +use shared::{ + model::{ + CurationConfigDto, PlaylistItem, PlaylistItemHeader, PlaylistItemType, SeriesStreamProperties, + StreamProperties, VideoStreamProperties, XtreamCluster, + }, + utils::{hash_string, Internable}, +}; + +fn catalog() -> Vec { + let mut groups = Vec::new(); + for (label, id, series) in [("movie", 7, false), ("alias", 7, false), ("other", 8, false), ("show", 7, true)] { + let cluster = if series { XtreamCluster::Series } else { XtreamCluster::Video }; + groups.push(PlaylistGroup { + id: 0, + title: label.intern(), + xtream_cluster: cluster, + channels: vec![PlaylistItem { + header: PlaylistItemHeader { + uuid: hash_string(label), + id: label.intern(), + title: label.intern(), + name: label.intern(), + group: label.intern(), + xtream_cluster: cluster, + item_type: if series { PlaylistItemType::SeriesInfo } else { PlaylistItemType::Video }, + additional_properties: Some(if series { + StreamProperties::Series(Box::new(SeriesStreamProperties { + tmdb: Some(id), + ..SeriesStreamProperties::default() + })) + } else { + StreamProperties::Video(Box::new(VideoStreamProperties { + tmdb: Some(id), + ..VideoStreamProperties::default() + })) + }), + ..PlaylistItemHeader::default() + }, + }], + }); + } + groups +} + +fn config(trakt_url: Option<&str>) -> CurationConfig { + let mut value = json!({"tmdb": {"api": {"access_token": "test-token"}, "trending": [ + {"kind": "movie", "time_window": "week", "limit": 1, "category_name": "TMDB Movies"}, + {"kind": "tv", "time_window": "day", "limit": 1, "category_name": "TMDB TV"} + ]}}); + if let Some(url) = trakt_url { + value["trakt"] = json!({"api": {"api_key": "test-client", "url": url}, "charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt", "tmdb_only": true}]}); + } + CurationConfig::from(&serde_json::from_value::(value).unwrap()) +} + +async fn run(config: &CurationConfig, playlist: &[PlaylistGroup], server: &TestServer) -> CurationRunOutcome { + let http = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .build() + .unwrap(); + let client = config.tmdb.as_ref().map(|source| TmdbClient::for_test(&http, &source.api, &server.url)); + evaluate_with_tmdb(&http, playlist, "test", config, client).await +} + +const PAGE: &str = r#"{"page":1,"total_pages":10,"total_results":100,"results":[{"id":7}]}"#; +const TRAKT: &str = r#"[{"title":"Film","year":2024,"ids":{"tmdb":7,"trakt":1,"slug":"film"}}]"#; + +#[tokio::test] +async fn tmdb_only_selects_exact_same_kind_subjects_and_retains_local_aliases() { + let server = TestServer::new(http_response(200, PAGE)).await; + let config = config(None); + let playlist = catalog(); + let CurationRunOutcome::Complete(result) = run(&config, &playlist, &server).await else { + panic!("TMDB-only succeeds without Trakt identity") + }; + assert_eq!(result.selectors.iter().map(|s| (s.key.0, s.membership_count)).collect::>(), [(0, 2), (1, 1)]); + assert_eq!( + result.memberships.iter().map(|m| (m.subject_uuid, m.media_kind)).collect::>(), + [ + (hash_string("movie"), CurationMediaKind::Movie), + (hash_string("alias"), CurationMediaKind::Movie), + (hash_string("show"), CurationMediaKind::Series), + ] + ); + let groups = project_curation_categories(&result, &playlist, &config); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].title.as_ref(), "TMDB Movies"); + assert_eq!( + groups[0].channels[0].header.uuid, + hash_string(&format!("trakt-category:TMDB Movies:{}", hash_string("movie"))) + ); + assert_eq!(groups[1].xtream_cluster, XtreamCluster::Series); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} + +#[tokio::test] +async fn mixed_sources_use_unique_keys_and_the_same_pre_projection_snapshot() { + let trakt = TestServer::new(http_response(200, TRAKT)).await; + let tmdb = TestServer::new(http_response(200, PAGE)).await; + let config = config(Some(&trakt.url)); + let playlist = catalog(); + let CurationRunOutcome::Complete(result) = run(&config, &playlist, &tmdb).await else { + panic!("both sources succeed") + }; + assert_eq!( + result.selectors.iter().map(|s| (s.key.0, s.membership_count)).collect::>(), + [(0, 2), (1, 2), (2, 1)] + ); + assert_eq!(result.memberships.len(), 5, "later adapters must not match earlier projections"); + let groups = project_curation_categories(&result, &playlist, &config); + assert_eq!(groups.iter().map(|g| g.title.as_ref()).collect::>(), ["Trakt", "TMDB Movies", "TMDB TV"]); + assert_ne!(groups[0].channels[0].header.uuid, groups[1].channels[0].header.uuid); + assert_eq!(trakt.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn either_failed_provider_blocks_the_whole_run_under_both_policies() { + for curated in [false, true] { + for tmdb_fails in [false, true] { + let trakt = TestServer::new(http_response(if tmdb_fails { 200 } else { 503 }, TRAKT)).await; + let tmdb = TestServer::new(http_response(if tmdb_fails { 503 } else { 200 }, PAGE)).await; + let mut config = config(Some(&trakt.url)); + if curated { + config.catalog_selection = shared::model::CurationCatalogSelection::Curated; + } + let CurationRunOutcome::Failed(failure) = run(&config, &catalog(), &tmdb).await else { + panic!("successful siblings are not publishable partial data") + }; + assert_eq!(failure.selector_outcomes.len(), 3); + assert_eq!( + failure.selector_outcomes.iter().filter(|o| matches!(o, SelectorOutcome::Complete { .. })).count(), + if tmdb_fails { 1 } else { 2 } + ); + } + } +} + +#[tokio::test] +async fn empty_and_no_match_are_complete_but_disabled_sources_are_not_configured() { + for body in [ + r#"{"page":1,"total_pages":0,"total_results":0,"results":[]}"#, + r#"{"page":1,"total_pages":1,"total_results":1,"results":[{"id":999,"title":"movie","name":"show"}]}"#, + ] { + let server = TestServer::new(http_response(200, body)).await; + let CurationRunOutcome::Complete(result) = run(&config(None), &catalog(), &server).await else { + panic!("valid empty or no-match is complete") + }; + assert!(result.memberships.is_empty()); + assert_eq!(result.selectors.len(), 2); + } + let server = TestServer::new(http_response(200, PAGE)).await; + let mut disabled = config(None); + disabled.enabled = false; + assert_eq!(run(&disabled, &catalog(), &server).await, CurationRunOutcome::NotConfigured); + disabled.enabled = true; + disabled.tmdb.as_mut().unwrap().enabled = false; + assert_eq!(run(&disabled, &catalog(), &server).await, CurationRunOutcome::NotConfigured); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn missing_credentials_mark_every_required_selector_with_its_assigned_key() { + let server = TestServer::new(http_response(200, PAGE)).await; + let mut config = config(Some(&server.url)); + config.trakt.as_mut().unwrap().api.api_key.clear(); + config.tmdb.as_mut().unwrap().api.access_token.clear(); + let CurationRunOutcome::Failed(result) = run(&config, &catalog(), &server).await else { + panic!("credentials unavailable") + }; + assert_eq!( + result.selector_outcomes, + (0..3) + .map(|key| SelectorOutcome::Unavailable { + key: CurationSelectorKey(key), + reason: CurationUnavailableReason::Configuration + }) + .collect::>() + ); + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn missing_tmdb_transport_marks_all_required_selectors_unavailable() { + let config = config(None); + let http = reqwest::Client::new(); + let result = evaluate_with_tmdb(&http, &catalog(), "test", &config, None).await; + let expected = CurationRunOutcome::Failed(CurationFailure { + selector_outcomes: (0..2) + .map(|key| SelectorOutcome::Unavailable { + key: CurationSelectorKey(key), + reason: CurationUnavailableReason::Configuration, + }) + .collect(), + }); + assert_eq!(result, expected); +} diff --git a/backend/curation/src/lib.rs b/backend/curation/src/lib.rs index 30efee45f..a45bb35d4 100644 --- a/backend/curation/src/lib.rs +++ b/backend/curation/src/lib.rs @@ -2,15 +2,20 @@ //! //! This crate owns the trusted, source-neutral matching and virtual-category //! projection kernel, plus the concrete edge adapter that translates foreign -//! source data before invoking that kernel. Trakt is currently the only adapter. +//! source data before invoking that kernel. Trakt and TMDB Trending are concrete adapters. //! //! Serialized configuration remains in `shared`, resolved configuration remains //! in `tuliprox-core`, and target-stage orchestration and persistence remain in //! their existing processing and repository crates. +mod coordinator; mod kernel; +#[cfg(test)] +mod test_support; +mod tmdb; mod trakt; +pub use coordinator::{evaluate_curation, project_curation_categories}; pub use kernel::{ CurationEvaluation, CurationFailure, CurationIncompleteReason, CurationMediaKind, CurationMembership, CurationRunOutcome, CurationSelectorKey, CurationSelectorSummary, CurationUnavailableReason, SelectorOutcome, diff --git a/backend/curation/src/test_support.rs b/backend/curation/src/test_support.rs new file mode 100644 index 000000000..e4d6b7cff --- /dev/null +++ b/backend/curation/src/test_support.rs @@ -0,0 +1,118 @@ +use std::{ + future::{poll_fn, Future}, + sync::{Arc, Mutex}, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::mpsc, + task::JoinHandle, +}; + +pub(crate) fn http_response(status: u16, body: &str) -> String { + format!("HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()) +} + +pub(crate) struct TestServer { + pub(crate) url: String, + pub(crate) requests: Arc>>, + task: JoinHandle<()>, +} + +impl TestServer { + pub(crate) async fn new(response: String) -> Self { Self::new_bytes(response.into_bytes()).await } + + pub(crate) async fn new_bytes(response: Vec) -> Self { + Self::delayed_bytes(response, Duration::ZERO, false).await + } + + pub(crate) async fn delayed_bytes(response: Vec, delay: Duration, send_headers: bool) -> Self { + Self::serve(vec![response], delay, send_headers).await + } + + pub(crate) async fn sequence(responses: Vec) -> Self { + Self::serve(responses.into_iter().map(String::into_bytes).collect(), Duration::ZERO, false).await + } + + pub(crate) async fn serve(responses: Vec>, delay: Duration, send_headers: bool) -> Self { + assert!(!responses.is_empty()); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let request = read_request(&mut stream).await; + let index = { + let mut requests = recorded.lock().unwrap(); + requests.push(request); + requests.len() - 1 + }; + let response = &responses[index.min(responses.len() - 1)]; + let split = if send_headers { + response.windows(4).position(|bytes| bytes == b"\r\n\r\n").unwrap() + 4 + } else { + 0 + }; + let _ = stream.write_all(&response[..split]).await; + tokio::time::sleep(delay).await; + let _ = stream.write_all(&response[split..]).await; + } + }); + Self { url, requests, task } + } + + /// Each received stream is a request-observed milestone. The test owns when + /// headers/body/chunks are released, and keeps stalled streams open until timeout. + pub(crate) async fn controlled() -> (Self, mpsc::UnboundedReceiver) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let (send, receive) = mpsc::unbounded_channel(); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let request = read_request(&mut stream).await; + recorded.lock().unwrap().push(request); + if send.send(stream).is_err() { + break; + } + } + }); + (Self { url, requests, task }, receive) + } +} + +async fn read_request(stream: &mut TcpStream) -> String { + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let count = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut buffer)).await.unwrap().unwrap(); + assert!(count > 0, "connection closed before request headers"); + request.extend_from_slice(&buffer[..count]); + assert!(request.len() <= 16384); + } + String::from_utf8(request).unwrap() +} + +/// Keep the paused runtime ready while real I/O progresses, preventing automatic +/// clock advance. This real-time watchdog also bounds missing milestones. All +/// acquisition/controller futures are inline, so panic drops them, not detached tasks. +pub(crate) async fn with_paused_io(future: impl Future) -> T { + let started = std::time::Instant::now(); + let mut future = std::pin::pin!(future); + poll_fn(|cx| { + assert!(started.elapsed() < Duration::from_secs(5), "paused-I/O test stalled (real-time watchdog)"); + let result = future.as_mut().poll(cx); + if result.is_pending() { + cx.waker().wake_by_ref(); + } + result + }) + .await +} + +impl Drop for TestServer { + fn drop(&mut self) { self.task.abort(); } +} diff --git a/backend/curation/src/tmdb/client.rs b/backend/curation/src/tmdb/client.rs new file mode 100644 index 000000000..5e52e4b98 --- /dev/null +++ b/backend/curation/src/tmdb/client.rs @@ -0,0 +1,257 @@ +use super::model::translate_page; +use crate::kernel::CuratedMediaReference; +use reqwest::{ + header::{HeaderValue, ACCEPT, AUTHORIZATION}, + Client, Url, +}; +use shared::model::{is_valid_tmdb_trending_limit, TmdbTrendingKind, TmdbTrendingTimeWindow}; +use std::{ + collections::HashSet, + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; +use tokio::{ + io::{AsyncRead, ReadBuf}, + time::{timeout_at, Instant}, +}; +use tuliprox_core::{ + model::{TmdbCurationApiConfig, TmdbTrendingConfig}, + utils::network::content_coding::{decode_complete_response_to_identity, read_to_end_limited, ContentBodyReadError}, +}; + +const TMDB_ORIGIN: &str = "https://api.themoviedb.org/"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); +const BODY_LIMIT: usize = 1024 * 1024; + +/// No response text, dynamic URL, headers or underlying errors that could echo credentials. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum TmdbFailure { + Configuration, + Transport, + Status(u16), + Redirect, + Body, + BodyLimit, + Deadline, + RequestLimit, + NoProgress, + InvalidResponse, +} + +/// Internal acquisition guards, independent of the configured reference count. +#[derive(Clone, Copy)] +struct Limits { + request_timeout: Duration, + request_bytes: usize, + selector_timeout: Duration, + selector_bytes: usize, + selector_requests: usize, + batch_timeout: Duration, + batch_bytes: usize, + batch_requests: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + request_timeout: REQUEST_TIMEOUT, + request_bytes: BODY_LIMIT, + selector_timeout: Duration::from_mins(1), + selector_bytes: 8 * BODY_LIMIT, + selector_requests: 32, + batch_timeout: Duration::from_mins(3), + batch_bytes: 32 * BODY_LIMIT, + batch_requests: 128, + } + } +} + +pub(super) struct AcquisitionBudget { + deadline: Instant, + requests: usize, + bytes: usize, + max_requests: usize, + max_bytes: usize, +} + +impl AcquisitionBudget { + fn new(duration: Duration, max_requests: usize, max_bytes: usize) -> Self { + Self { deadline: Instant::now() + duration, requests: 0, bytes: 0, max_requests, max_bytes } + } + + fn check_time(&self) -> Result<(), TmdbFailure> { + if Instant::now() >= self.deadline { + Err(TmdbFailure::Deadline) + } else { + Ok(()) + } + } + + fn admit(&self) -> Result<(), TmdbFailure> { + self.check_time()?; + if self.requests >= self.max_requests { + return Err(TmdbFailure::RequestLimit); + } + if self.bytes >= self.max_bytes { + return Err(TmdbFailure::BodyLimit); + } + Ok(()) + } +} + +/// Account at the reader boundary, including partial failed bodies and the exact/+1 probe. +/// The outer timeout may drop a read future; already consumed bytes remain debited. +struct CountedReader<'a, R> { + inner: R, + selector_bytes: &'a mut usize, + batch_bytes: &'a mut usize, +} + +impl AsyncRead for CountedReader<'_, R> { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let this = self.get_mut(); + let before = buf.filled().len(); + let result = Pin::new(&mut this.inner).poll_read(cx, buf); + let consumed = buf.filled().len() - before; + *this.selector_bytes += consumed; + *this.batch_bytes += consumed; + #[cfg(test)] + tests::record_consumed_bytes(consumed); + result + } +} + +pub(crate) struct TmdbClient { + http: Client, + authorization: HeaderValue, + origin: Url, + limits: Limits, +} + +impl TmdbClient { + /// `http` must be the configured TMDB profile, never the generic/Trakt client. + pub(crate) fn new(http: &Client, api: &TmdbCurationApiConfig) -> Result { + let token = api.access_token.trim(); + if token.is_empty() { + return Err(TmdbFailure::Configuration); + } + let mut authorization = + HeaderValue::from_str(&format!("Bearer {token}")).map_err(|_| TmdbFailure::Configuration)?; + authorization.set_sensitive(true); + Ok(Self { + http: http.clone(), + authorization, + origin: Url::parse(TMDB_ORIGIN).map_err(|_| TmdbFailure::Configuration)?, + limits: Limits::default(), + }) + } + + #[cfg(test)] + pub(crate) fn for_test(http: &Client, api: &TmdbCurationApiConfig, origin: &str) -> Result { + let mut client = Self::new(http, api)?; + client.origin = Url::parse(origin).map_err(|_| TmdbFailure::Configuration)?; + Ok(client) + } + + pub(super) fn batch_budget(&self) -> AcquisitionBudget { + AcquisitionBudget::new(self.limits.batch_timeout, self.limits.batch_requests, self.limits.batch_bytes) + } + + pub(super) async fn trending( + &self, + selector: &TmdbTrendingConfig, + batch: &mut AcquisitionBudget, + ) -> Result, TmdbFailure> { + if !is_valid_tmdb_trending_limit(selector.limit) { + return Err(TmdbFailure::Configuration); + } + let mut budget = AcquisitionBudget::new( + self.limits.selector_timeout, + self.limits.selector_requests, + self.limits.selector_bytes, + ); + let kind = match selector.kind { + TmdbTrendingKind::Movie => "movie", + TmdbTrendingKind::Tv => "tv", + }; + let window = match selector.time_window { + TmdbTrendingTimeWindow::Day => "day", + TmdbTrendingTimeWindow::Week => "week", + }; + let mut page_number = 1_u64; + let mut preceding_rows = 0_u32; + let mut seen = HashSet::new(); + let mut references = Vec::new(); + loop { + budget.admit()?; + batch.admit()?; + let deadline = (Instant::now() + self.limits.request_timeout).min(budget.deadline).min(batch.deadline); + let max_bytes = + self.limits.request_bytes.min(budget.max_bytes - budget.bytes).min(batch.max_bytes - batch.bytes); + let mut url = + self.origin.join(&format!("3/trending/{kind}/{window}")).map_err(|_| TmdbFailure::Configuration)?; + url.query_pairs_mut().append_pair("language", "en-US").append_pair("page", &page_number.to_string()); + // Debit before sending, including transport failures. The profile forbids redirects/replays. + budget.requests += 1; + batch.requests += 1; + let body = timeout_at(deadline, async { + let response = self + .http + .get(url.clone()) + .header(AUTHORIZATION, self.authorization.clone()) + .header(ACCEPT, "application/json") + .send() + .await + .map_err(|_| TmdbFailure::Transport)?; + if response.status().is_redirection() || response.url() != &url { + return Err(TmdbFailure::Redirect); + } + if !response.status().is_success() { + return Err(TmdbFailure::Status(response.status().as_u16())); + } + let decoded = decode_complete_response_to_identity(response).await.map_err(|_| TmdbFailure::Body)?; + let mut reader = CountedReader { + inner: decoded.body, + selector_bytes: &mut budget.bytes, + batch_bytes: &mut batch.bytes, + }; + read_to_end_limited(&mut reader, max_bytes).await.map_err(|error| match error { + ContentBodyReadError::LimitExceeded { .. } => TmdbFailure::BodyLimit, + ContentBodyReadError::InvalidUtf8 { .. } | ContentBodyReadError::Io(_) => TmdbFailure::Body, + }) + }) + .await + .map_err(|_| TmdbFailure::Deadline)??; + let page = translate_page(&body, selector.kind, page_number, preceding_rows) + .map_err(|()| TmdbFailure::InvalidResponse)?; + preceding_rows = preceding_rows + .checked_add(u32::try_from(page.rows.len()).map_err(|_| TmdbFailure::InvalidResponse)?) + .ok_or(TmdbFailure::InvalidResponse)?; + let before = references.len(); + for reference in page.rows { + let id = reference.tmdb_id.ok_or(TmdbFailure::InvalidResponse)?; + if seen.insert(id) { + references.push(reference); + if references.len() == selector.limit as usize { + break; + } + } + } + // Deadlines include wire parsing/normalization, not local matching or publication. + if Instant::now() >= deadline { + return Err(TmdbFailure::Deadline); + } + if page_number > 1 && references.len() == before { + return Err(TmdbFailure::NoProgress); + } + if references.len() == selector.limit as usize || page.last { + return Ok(references); + } + page_number = page_number.checked_add(1).ok_or(TmdbFailure::InvalidResponse)?; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/backend/curation/src/tmdb/client/tests.rs b/backend/curation/src/tmdb/client/tests.rs new file mode 100644 index 000000000..14b071435 --- /dev/null +++ b/backend/curation/src/tmdb/client/tests.rs @@ -0,0 +1,665 @@ +use super::*; +use crate::test_support::{http_response, with_paused_io, TestServer}; +use serde_json::{json, Value}; +use std::cell::Cell; +use tokio::io::AsyncWriteExt; +use tuliprox_core::utils::compression_utils::compress_string; + +const EMPTY: &str = r#"{"page":1,"total_pages":0,"total_results":0,"results":[]}"#; +const TOKEN: &str = "private-test-token"; + +// The paused tests run on one thread. Observe the existing reader's byte debit, +// not a socket write: only consumption is a valid milestone before advancing time. +thread_local! { + static CONSUMED_BYTES: Cell = const { Cell::new(0) }; +} + +pub(super) fn record_consumed_bytes(bytes: usize) { CONSUMED_BYTES.with(|count| count.set(count.get() + bytes)); } + +async fn wait_for_consumed_bytes(expected: usize) { + while CONSUMED_BYTES.with(Cell::get) < expected { + tokio::task::yield_now().await; + } + assert_eq!(CONSUMED_BYTES.with(Cell::get), expected); +} + +fn selector(limit: u32) -> TmdbTrendingConfig { + TmdbTrendingConfig { + kind: TmdbTrendingKind::Movie, + time_window: TmdbTrendingTimeWindow::Week, + limit, + category_name: None, + create_xtream_category: false, + } +} + +fn client(server: &TestServer) -> TmdbClient { + // Adapter tests; the production configured profile is exercised independently in + // core's transport fixtures and in HTTPS -> processing publication tests. + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert("x-transport-context", HeaderValue::from_static("retained")); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .default_headers(headers) + .build() + .unwrap(); + TmdbClient::for_test(&http, &TmdbCurationApiConfig { access_token: TOKEN.into() }, &server.url).unwrap() +} + +impl TmdbClient { + async fn fetch(&self, selector: &TmdbTrendingConfig) -> Result, TmdbFailure> { + self.trending(selector, &mut self.batch_budget()).await + } +} + +fn page(p: u64, total_pages: u64, ids: &[u32]) -> String { + json!({"page":p,"total_pages":total_pages,"total_results":ids.len(),"results":ids.iter().map(|id| json!({"id":id})).collect::>()}).to_string() +} + +fn gzip_response(body: &str) -> Vec { encoded_response(&compress_string(body).unwrap()) } + +fn encoded_response(bytes: &[u8]) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ) + .into_bytes(); + response.extend_from_slice(bytes); + response +} + +fn padded(body: &str, length: usize) -> String { format!("{body}{}", " ".repeat(length - body.len())) } + +#[tokio::test] +async fn tmdb_item_limit_cuts_unique_references_with_first_text_and_observed_row_rank() { + for (limit, expected_ids, expected_ranks, requests) in [ + (1, vec![7], vec![1], 1), + (2, vec![7, 8], vec![1, 3], 1), + (3, vec![7, 8, 9], vec![1, 3, 5], 2), + (4, vec![7, 8, 9, 10], vec![1, 3, 5, 6], 2), + (100, vec![7, 8, 9, 10], vec![1, 3, 5, 6], 2), + ] { + let first = r#"{"page":1,"total_pages":9,"total_results":10,"results":[{"id":7,"title":"First"},{"id":7,"title":"Duplicate"},{"id":8}]}"#; + let server = + TestServer::sequence(vec![http_response(200, first), http_response(200, &page(2, 2, &[8, 9, 10]))]).await; + let result = client(&server).fetch(&selector(limit)).await.unwrap(); + assert_eq!(result.iter().map(|r| r.tmdb_id.unwrap()).collect::>(), expected_ids); + assert_eq!(result.iter().map(|r| r.rank.unwrap()).collect::>(), expected_ranks); + assert_eq!(result[0].title, "First"); + let recorded = server.requests.lock().unwrap(); + assert_eq!(recorded.len(), requests); + for (index, request) in recorded.iter().enumerate() { + assert!( + request.starts_with(&format!("GET /3/trending/movie/week?language=en-US&page={} HTTP/1.1", index + 1)) + ); + assert!(!request.lines().next().unwrap().contains("limit")); + } + } +} + +#[tokio::test] +async fn tmdb_short_pages_and_drifting_totals_do_not_assume_a_snapshot() { + let server = TestServer::sequence(vec![ + http_response(200, &page(1, 2, &[7])), + http_response(200, &page(2, 5, &[8, 9, 10, 11])), + http_response(200, &page(3, 3, &[12])), + ]) + .await; + let result = client(&server).fetch(&selector(100)).await.unwrap(); + assert_eq!(result.len(), 6); + assert_eq!(result[5].rank, Some(6)); + assert_eq!(server.requests.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn tmdb_empty_is_valid_only_at_the_start_with_coherent_totals() { + for pages in [0, 1] { + let body = json!({"page":1,"total_pages":pages,"total_results":0,"results":[]}).to_string(); + let server = TestServer::new(http_response(200, &body)).await; + assert!(client(&server).fetch(&selector(100)).await.unwrap().is_empty()); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } + for body in [ + page(2, 2, &[]), + page(2, 9, &[]), + EMPTY.into(), + json!({"page":2,"total_pages":2,"total_results":1,"results":[]}).to_string(), + ] { + let server = TestServer::sequence(vec![http_response(200, &page(1, 2, &[7])), http_response(200, &body)]).await; + assert_eq!(client(&server).fetch(&selector(100)).await.unwrap_err(), TmdbFailure::InvalidResponse); + assert_eq!(server.requests.lock().unwrap().len(), 2); + } +} + +#[tokio::test] +async fn tmdb_no_progress_including_terminal_permutations_fails_instead_of_accepting_a_prefix() { + for pages in [2, 3] { + for ids in [&[7, 8][..], &[8, 7], &[7, 7]] { + let server = TestServer::sequence(vec![ + http_response(200, &page(1, 3, &[7, 8])), + http_response(200, &page(2, pages, ids)), + ]) + .await; + assert_eq!(client(&server).fetch(&selector(100)).await.unwrap_err(), TmdbFailure::NoProgress); + assert_eq!(server.requests.lock().unwrap().len(), 2); + } + } +} + +#[tokio::test] +async fn tmdb_validates_envelope_duplicates_and_suffix_even_after_the_limit() { + let mut invalid: Vec = vec![json!({}), json!([])]; + let base = json!({"page":1,"total_pages":1,"total_results":2,"results":[{"id":7},{"id":8}]}); + for (field, value) in [ + ("page", json!(2)), + ("page", json!(0)), + ("page", json!(1.0)), + ("total_pages", json!(0)), + ("total_results", json!(1)), + ("total_results", json!(-1)), + ("results", json!(null)), + ] { + let mut value_base = base.clone(); + value_base[field] = value; + invalid.push(value_base); + } + for field in ["page", "total_pages", "total_results", "results"] { + let mut value = base.clone(); + value.as_object_mut().unwrap().remove(field); + invalid.push(value); + } + for bad in [ + json!({"id":0}), + json!({"id":7,"media_type":"tv"}), + json!({"id":7,"media_type":null}), + json!({"id":7,"title":42}), + ] { + for index in [0, 1] { + let mut value = base.clone(); + value["results"][index] = bad.clone(); + invalid.push(value); + } + } + for value in invalid { + let server = TestServer::new(http_response(200, &value.to_string())).await; + assert_eq!(client(&server).fetch(&selector(1)).await.unwrap_err(), TmdbFailure::InvalidResponse, "{value}"); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } + // A current envelope below the requested page is invalid even at N. + let server = + TestServer::sequence(vec![http_response(200, &page(1, 2, &[7])), http_response(200, &page(2, 1, &[8]))]).await; + assert_eq!(client(&server).fetch(&selector(2)).await.unwrap_err(), TmdbFailure::InvalidResponse); +} + +#[tokio::test] +async fn tmdb_all_routes_preserve_headers_with_no_query_token_or_details() { + for (kind, route) in [(TmdbTrendingKind::Movie, "movie"), (TmdbTrendingKind::Tv, "tv")] { + for (window, period) in [(TmdbTrendingTimeWindow::Day, "day"), (TmdbTrendingTimeWindow::Week, "week")] { + let server = TestServer::new(http_response(200, EMPTY)).await; + let client = client(&server); + assert!(client.authorization.is_sensitive()); + assert!(!format!("{:?}", client.authorization).contains(TOKEN)); + let mut selector = selector(100); + selector.kind = kind; + selector.time_window = window; + assert!(client.fetch(&selector).await.unwrap().is_empty()); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!( + requests[0].starts_with(&format!("GET /3/trending/{route}/{period}?language=en-US&page=1 HTTP/1.1")) + ); + assert!(requests[0].to_lowercase().contains(&format!("authorization: bearer {TOKEN}"))); + assert!(requests[0].contains("x-transport-context: retained")); + } + } +} + +#[tokio::test] +async fn tmdb_invalid_runtime_limit_and_unavailable_credentials_send_nothing() { + let server = TestServer::new(http_response(200, EMPTY)).await; + for token in ["", " ", "bad\nheader"] { + assert!(matches!( + TmdbClient::for_test(&Client::new(), &TmdbCurationApiConfig { access_token: token.into() }, &server.url), + Err(TmdbFailure::Configuration) + )); + } + for limit in [0, 501, u32::MAX] { + assert_eq!(client(&server).fetch(&selector(limit)).await.unwrap_err(), TmdbFailure::Configuration); + } + assert!(server.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn tmdb_status_failures_are_sanitized_without_retry_or_body_consumption() { + for status in [401, 403, 429, 500, 503] { + for late in [false, true] { + let mut responses = Vec::new(); + if late { + responses.push(http_response(200, &page(1, 2, &[7]))); + } + responses.push(http_response(status, TOKEN)); + let server = TestServer::sequence(responses).await; + let client = client(&server); + let mut budget = client.batch_budget(); + let error = client.trending(&selector(100), &mut budget).await.unwrap_err(); + assert_eq!(error, TmdbFailure::Status(status)); + assert!(!format!("{error:?}").contains(TOKEN)); + assert_eq!(budget.bytes, if late { page(1, 2, &[7]).len() } else { 0 }); + assert_eq!(budget.requests, if late { 2 } else { 1 }); + assert_eq!(server.requests.lock().unwrap().len(), budget.requests); + } + } +} + +#[tokio::test] +async fn tmdb_identity_chunked_and_gzip_exact_and_plus_one_count_decoded_bytes() { + for length in [BODY_LIMIT, BODY_LIMIT + 1] { + let body = padded(EMPTY, length); + let chunked = format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n{:x}\r\n{body}\r\n0\r\n\r\n", + body.len() + ); + for response in [http_response(200, &body).into_bytes(), chunked.into_bytes(), gzip_response(&body)] { + let server = TestServer::new_bytes(response).await; + let client = client(&server); + let mut budget = client.batch_budget(); + let result = client.trending(&selector(100), &mut budget).await; + if length == BODY_LIMIT { + assert!(result.unwrap().is_empty()); + } else { + assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit); + } + assert_eq!(budget.bytes, length); + } + } +} + +#[tokio::test] +async fn tmdb_complete_gzip_member_does_not_hide_truncated_transport_or_later_members() { + let compressed = compress_string(EMPTY).unwrap(); + let mut truncated_transport = format!( + "HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + compressed.len() + 1 + ) + .into_bytes(); + truncated_transport.extend_from_slice(&compressed); + let mut broken_second_member = compressed.clone(); + let mut suffix = compress_string(" ").unwrap(); + suffix.truncate(suffix.len() - 8); + broken_second_member.extend(suffix); + for response in [truncated_transport, encoded_response(&broken_second_member)] { + let server = TestServer::new_bytes(response).await; + assert_eq!(client(&server).fetch(&selector(1)).await.unwrap_err(), TmdbFailure::Body); + } + let mut oversized_members = compressed; + oversized_members.extend(compress_string(&" ".repeat(BODY_LIMIT + 1 - EMPTY.len())).unwrap()); + let server = TestServer::new_bytes(encoded_response(&oversized_members)).await; + let client = client(&server); + let mut budget = client.batch_budget(); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + assert_eq!(budget.bytes, BODY_LIMIT + 1); +} + +#[tokio::test] +async fn tmdb_failed_bodies_and_detection_probes_are_debited_before_a_sibling() { + let mut compressed = compress_string(&padded(EMPTY, 1000)).unwrap(); + compressed.truncate(compressed.len() - 8); // full decoded JSON, missing gzip trailer + for response in [ + encoded_response(&compressed), + format!("HTTP/1.1 200 OK\r\nContent-Length: 2000\r\nConnection: close\r\n\r\n{}", padded(EMPTY, 1000)) + .into_bytes(), + ] { + let server = TestServer::new_bytes(response).await; + let mut client = client(&server); + client.limits.batch_bytes = 1000; + let mut budget = client.batch_budget(); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::Body); + assert_eq!(budget.bytes, 1000); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } + let server = TestServer::new(http_response(200, &padded(EMPTY, 1001))).await; + let mut client = client(&server); + client.limits.batch_bytes = 1000; + let mut budget = client.batch_budget(); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + assert_eq!(budget.bytes, 1001); + assert!(client.trending(&selector(1), &mut budget).await.is_err()); + assert_eq!(server.requests.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn tmdb_selector_and_shared_batch_bytes_accept_exact_completion_but_not_excess() { + for batch_bound in [false, true] { + for excess in [0, 1] { + let server = TestServer::sequence(vec![ + http_response(200, &padded(&page(1, 2, &[7]), 200)), + http_response(200, &padded(&page(2, 2, &[8]), 200 + excess)), + ]) + .await; + let mut client = client(&server); + if batch_bound { + client.limits.batch_bytes = 400; + } else { + client.limits.selector_bytes = 400; + } + let mut budget = client.batch_budget(); + let result = client.trending(&selector(100), &mut budget).await; + if excess == 0 { + assert_eq!(result.unwrap().len(), 2); + } else { + assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit); + } + assert_eq!(budget.bytes, 400 + excess); + assert_eq!(server.requests.lock().unwrap().len(), 2); + } + } + let server = TestServer::new(http_response(200, &padded(EMPTY, 200))).await; + let mut client = client(&server); + client.limits.batch_bytes = 400; + let mut budget = client.batch_budget(); + for _ in 0..2 { + client.trending(&selector(1), &mut budget).await.unwrap(); + } + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + assert_eq!(budget.bytes, 400); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} + +#[tokio::test] +async fn tmdb_production_8_and_32_mib_guards_count_all_decoded_pages_and_one_detection_byte() { + for selectors in [1, 4] { + for excess in [0, 1] { + let mut responses = Vec::new(); + for index in 0..selectors { + for p in 1..=16 { + let size = BODY_LIMIT / 2 + if index == selectors - 1 && p == 16 { excess } else { 0 }; + responses.push(gzip_response(&padded(&page(p, 16, &[u32::try_from(p).unwrap()]), size))); + } + } + let server = TestServer::serve(responses, Duration::ZERO, false).await; + let client = client(&server); + let mut budget = client.batch_budget(); + for index in 0..selectors { + let result = client.trending(&selector(500), &mut budget).await; + if index == selectors - 1 && excess != 0 { + assert_eq!(result.unwrap_err(), TmdbFailure::BodyLimit); + } else { + assert_eq!(result.unwrap().len(), 16); + } + } + assert_eq!(budget.bytes, selectors * 8 * BODY_LIMIT + excess); + if selectors == 4 { + assert_eq!(client.trending(&selector(500), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + } + assert_eq!(server.requests.lock().unwrap().len(), selectors * 16); + } + } +} + +#[tokio::test] +async fn tmdb_request_guards_admit_32_and_128_but_never_33_or_129() { + for complete in [false, true] { + let responses = (1..=32) + .map(|p| http_response(200, &page(p, if complete { 32 } else { 33 }, &[u32::try_from(p).unwrap()]))) + .collect(); + let server = TestServer::sequence(responses).await; + let result = client(&server).fetch(&selector(500)).await; + if complete { + assert_eq!(result.unwrap().len(), 32); + } else { + assert_eq!(result.unwrap_err(), TmdbFailure::RequestLimit); + } + assert_eq!(server.requests.lock().unwrap().len(), 32); + } + let server = TestServer::new(http_response(200, EMPTY)).await; + let client = client(&server); + let mut budget = client.batch_budget(); + for _ in 0..128 { + client.trending(&selector(1), &mut budget).await.unwrap(); + } + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::RequestLimit); + assert_eq!(server.requests.lock().unwrap().len(), 128); +} + +#[tokio::test(start_paused = true)] +async fn tmdb_request_deadline_covers_headers_and_bodies() { + with_paused_io(async { + for headers in [false, true] { + let (server, mut received) = TestServer::controlled().await; + let mut client = client(&server); + client.limits.request_timeout = Duration::from_millis(50); + let mut budget = client.batch_budget(); + let selected = selector(1); + let (result, _stalled) = tokio::join!(client.trending(&selected, &mut budget), async { + let mut stream = received.recv().await.unwrap(); + if headers { + // Consume a useful prefix, proving headers have been processed, + // but keep the declared body incomplete until the request deadline. + let before = CONSUMED_BYTES.with(Cell::get); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n{").await.unwrap(); + wait_for_consumed_bytes(before + 1).await; + } + tokio::time::advance(Duration::from_millis(50)).await; + stream + }); + assert_eq!(result.unwrap_err(), TmdbFailure::Deadline); + assert_eq!(budget.requests, 1); + assert_eq!(budget.bytes, usize::from(headers)); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } + }) + .await; +} + +async fn accumulated_deadline(batch_bound: bool) { + with_paused_io(async { + let (server, mut received) = TestServer::controlled().await; + let mut client = client(&server); + client.limits.request_timeout = Duration::from_secs(1); + if batch_bound { + client.limits.batch_timeout = Duration::from_millis(110); + } else { + client.limits.selector_timeout = Duration::from_millis(110); + } + let mut budget = client.batch_budget(); + let selected = selector(100); + let (result, _stalled) = tokio::join!(client.trending(&selected, &mut budget), async { + for p in 1..=2 { + let mut stream = received.recv().await.unwrap(); + tokio::time::advance(Duration::from_millis(40)).await; + stream + .write_all(http_response(200, &page(p, 4, &[u32::try_from(p).unwrap()])).as_bytes()) + .await + .unwrap(); + } + // GET3 proves pages 1 and 2 were consumed/normalized at t=40/80, + // not merely written by the server. Keep page 3 incomplete at t=110. + let mut stream = received.recv().await.unwrap(); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n").await.unwrap(); + tokio::time::advance(Duration::from_millis(30)).await; + stream + }); + assert_eq!(result.unwrap_err(), TmdbFailure::Deadline); + assert_eq!(budget.requests, 3); + assert_eq!(budget.bytes, page(1, 4, &[1]).len() + page(2, 4, &[2]).len()); + if batch_bound { + assert_eq!(client.trending(&selected, &mut budget).await.unwrap_err(), TmdbFailure::Deadline); + assert_eq!(budget.requests, 3, "pending selector cannot reset the batch clock"); + } + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3, "no fourth page or pending selector traffic"); + for (index, request) in requests.iter().enumerate() { + assert!(request.starts_with(&format!("GET /3/trending/movie/week?language=en-US&page={} ", index + 1))); + } + }) + .await; +} + +#[tokio::test(start_paused = true)] +async fn tmdb_selector_deadline_accumulates_pages() { accumulated_deadline(false).await; } + +#[tokio::test(start_paused = true)] +async fn tmdb_batch_deadline_accumulates_pages_and_refuses_pending_selectors() { accumulated_deadline(true).await; } + +#[tokio::test(start_paused = true)] +async fn tmdb_slow_chunks_do_not_reset_deadline_or_accept_a_useful_json_prefix() { + with_paused_io(async { + let (server, mut received) = TestServer::controlled().await; + let mut client = client(&server); + client.limits.request_timeout = Duration::from_millis(100); + let mut budget = client.batch_budget(); + let selected = selector(1); + let before = CONSUMED_BYTES.with(Cell::get); + let (result, _stalled) = tokio::join!(client.trending(&selected, &mut budget), async { + let mut stream = received.recv().await.unwrap(); + let body = padded(&page(1, 1, &[7]), 1000); + assert!(serde_json::from_str::(&body[..100]).is_ok(), "first chunk is already useful JSON"); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n").await.unwrap(); + for (index, chunk) in body.as_bytes().chunks(100).take(3).enumerate() { + tokio::time::advance(Duration::from_millis(30)).await; + stream.write_all(chunk).await.unwrap(); + wait_for_consumed_bytes(before + (index + 1) * 100).await; + } + tokio::time::advance(Duration::from_millis(10)).await; + stream + }); + assert_eq!(result.unwrap_err(), TmdbFailure::Deadline); + assert_eq!(budget.bytes, 300, "consumed at t=30/60/90, never accepted as a complete body"); + assert_eq!(budget.requests, 1); + assert_eq!(server.requests.lock().unwrap().len(), 1); + }) + .await; +} + +#[tokio::test] +async fn tmdb_failed_selector_bytes_and_probe_reduce_the_next_selector_allowance() { + let server = TestServer::new(http_response(200, &padded(EMPTY, 1000))).await; + let mut client = client(&server); + client.limits.request_bytes = 128; + client.limits.batch_bytes = 256; + let mut budget = client.batch_budget(); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + assert_eq!(budget.bytes, 129); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::BodyLimit); + assert_eq!(budget.bytes, 257, "remaining 127 plus one detection byte, not a reset budget"); + assert!(client.trending(&selector(1), &mut budget).await.is_err()); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} + +#[tokio::test] +async fn tmdb_transport_failure_debits_the_admitted_attempt_without_resetting_the_batch() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}/", listener.local_addr().unwrap()); + drop(listener); + let http = Client::builder().no_proxy().retry(reqwest::retry::never()).build().unwrap(); + let mut client = + TmdbClient::for_test(&http, &TmdbCurationApiConfig { access_token: TOKEN.into() }, &origin).unwrap(); + client.limits.batch_requests = 1; + let mut budget = client.batch_budget(); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::Transport); + assert_eq!(budget.requests, 1); + assert_eq!(client.trending(&selector(1), &mut budget).await.unwrap_err(), TmdbFailure::RequestLimit); + assert_eq!(budget.requests, 1); +} + +#[tokio::test(start_paused = true)] +async fn tmdb_batch_deadline_blocks_admission_with_matching_prefix_and_completed_sibling() { + use crate::kernel::{CurationIncompleteReason, CurationRunOutcome, CurationSelectorKey, SelectorOutcome}; + use shared::{ + model::{ + CurationConfigDto, PlaylistGroup, PlaylistItem, PlaylistItemHeader, PlaylistItemType, StreamProperties, + VideoStreamProperties, XtreamCluster, + }, + utils::{hash_string, Internable}, + }; + let (server, mut received) = TestServer::controlled().await; + let mut client = client(&server); + client.limits.batch_timeout = Duration::from_millis(150); + let dto: CurationConfigDto = serde_json::from_value(json!({"tmdb":{"trending":[ + {"kind":"movie","time_window":"week","limit":1,"create_xtream_category":false}, + {"kind":"movie","time_window":"week","limit":100,"create_xtream_category":false}, + {"kind":"movie","time_window":"week","limit":1,"create_xtream_category":false} + ]}})) + .unwrap(); + let config = tuliprox_core::model::CurationConfig::from(&dto).tmdb.unwrap(); + let playlist = vec![PlaylistGroup { + id: 1, + title: "Movies".intern(), + xtream_cluster: XtreamCluster::Video, + channels: [7, 8] + .into_iter() + .map(|id| PlaylistItem { + header: PlaylistItemHeader { + uuid: hash_string(&format!("subject-{id}")), + title: format!("Movie {id}").intern(), + item_type: PlaylistItemType::Video, + xtream_cluster: XtreamCluster::Video, + additional_properties: Some(StreamProperties::Video(Box::new(VideoStreamProperties { + tmdb: Some(id), + ..Default::default() + }))), + ..Default::default() + }, + }) + .collect(), + }]; + let keys = [CurationSelectorKey(0), CurationSelectorKey(1), CurationSelectorKey(2)]; + let before = CONSUMED_BYTES.with(Cell::get); + let (outcomes, _stalled) = with_paused_io(async { + tokio::join!(crate::tmdb::evaluate_selectors(Ok(client), &playlist, "deadline", &config, &keys), async { + for body in [page(1, 1, &[7]), page(1, 9, &[8])] { + let mut stream = received.recv().await.unwrap(); + tokio::time::advance(Duration::from_millis(60)).await; + stream.write_all(http_response(200, &body).as_bytes()).await.unwrap(); + } + // Third GET proves sibling completion and a matching prefix in the next selector. + let mut stream = received.recv().await.unwrap(); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n").await.unwrap(); + tokio::time::advance(Duration::from_millis(30)).await; + stream + }) + }) + .await; + assert_eq!(CONSUMED_BYTES.with(Cell::get) - before, page(1, 1, &[7]).len() + page(1, 9, &[8]).len()); + assert!(matches!(&outcomes[0], SelectorOutcome::Complete { memberships, .. } if memberships.len() == 1)); + for outcome in &outcomes[1..] { + assert!(matches!(outcome, SelectorOutcome::Incomplete { reason: CurationIncompleteReason::Interrupted, .. })); + } + assert!(matches!(crate::coordinator::complete_evaluation(outcomes), CurationRunOutcome::Failed(_))); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3, "pending sibling does not reset the batch clock"); + for (request, page) in requests.iter().zip([1, 1, 2]) { + assert!(request.starts_with(&format!("GET /3/trending/movie/week?language=en-US&page={page} "))); + } +} + +#[tokio::test(start_paused = true)] +async fn tmdb_budget_clock_uses_absolute_not_sliding_deadlines() { + let budget = AcquisitionBudget::new(Duration::from_secs(60), 32, 8 * BODY_LIMIT); + tokio::time::advance(Duration::from_secs(59)).await; + budget.admit().unwrap(); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(budget.admit(), Err(TmdbFailure::Deadline)); +} + +#[tokio::test] +async fn tmdb_redirects_never_visit_same_or_foreign_destination() { + let foreign = TestServer::new(http_response(200, EMPTY)).await; + for location in [ + "/3/trending/movie/day?language=en-US&page=1".into(), + format!("{}foreign", foreign.url), + "http://api.themoviedb.org/".into(), + "/3/trending/movie/week?language=en-US&page=1".into(), + ] { + let server = TestServer::new(format!( + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )) + .await; + assert_eq!(client(&server).fetch(&selector(1)).await.unwrap_err(), TmdbFailure::Redirect); + assert_eq!(server.requests.lock().unwrap().len(), 1); + } + assert!(foreign.requests.lock().unwrap().is_empty()); +} diff --git a/backend/curation/src/tmdb/mod.rs b/backend/curation/src/tmdb/mod.rs new file mode 100644 index 000000000..642c65f42 --- /dev/null +++ b/backend/curation/src/tmdb/mod.rs @@ -0,0 +1,100 @@ +mod client; +mod model; + +use crate::{ + coordinator::LEGACY_CATEGORY_NAMESPACE, + kernel::{ + evaluate_selector, project_memberships, CurationCategorySpec, CurationEvaluation, CurationIncompleteReason, + CurationMatchPolicy, CurationMediaScope, CurationProjectionCatalog, CurationSelectorKey, CurationSelectorSpec, + CurationUnavailableReason, ProjectionIdentityStrategy, SelectorOutcome, + }, +}; +pub(crate) use client::{TmdbClient, TmdbFailure}; +use shared::model::{PlaylistGroup, TmdbTrendingKind}; +use tuliprox_core::model::TmdbCurationConfig; + +fn selector_spec(kind: TmdbTrendingKind) -> CurationSelectorSpec { + CurationSelectorSpec { + media_scope: match kind { + TmdbTrendingKind::Movie => CurationMediaScope::Movies, + TmdbTrendingKind::Tv => CurationMediaScope::Series, + }, + match_policy: CurationMatchPolicy::ExactTmdbOnly, + } +} + +pub(crate) async fn evaluate_selectors( + client: Result, + playlist: &[PlaylistGroup], + target: &str, + config: &TmdbCurationConfig, + keys: &[CurationSelectorKey], +) -> Vec { + assert_eq!(keys.len(), config.trending.len()); + // Acquire the whole TMDB batch before matching: deadlines bound acquisition and + // normalization only. Failed selectors never expose their selected prefix. + let mut batch = client.as_ref().ok().map(TmdbClient::batch_budget); + let mut acquired = Vec::with_capacity(keys.len()); + for selector in &config.trending { + acquired.push(match &client { + Ok(client) => match batch.as_mut() { + Some(batch) => client.trending(selector, batch).await, + None => Err(TmdbFailure::Configuration), + }, + Err(error) => Err(*error), + }); + } + let mut outcomes = Vec::with_capacity(keys.len()); + for ((selector, key), references) in config.trending.iter().zip(keys).zip(acquired) { + outcomes.push(match references { + Ok(references) => evaluate_selector(*key, &references, playlist, selector_spec(selector.kind)), + Err(error) => { + log::warn!("TMDB trending selector {} unavailable for target '{target}': {error:?}", key.0); + match error { + TmdbFailure::Configuration => { + SelectorOutcome::Unavailable { key: *key, reason: CurationUnavailableReason::Configuration } + } + TmdbFailure::RequestLimit | TmdbFailure::NoProgress => { + SelectorOutcome::Incomplete { key: *key, reason: CurationIncompleteReason::PaginationTruncated } + } + TmdbFailure::Body | TmdbFailure::BodyLimit | TmdbFailure::Deadline => { + SelectorOutcome::Incomplete { key: *key, reason: CurationIncompleteReason::Interrupted } + } + _ => SelectorOutcome::Unavailable { key: *key, reason: CurationUnavailableReason::Source }, + } + } + }); + } + outcomes +} + +pub(crate) fn project_categories( + evaluation: &CurationEvaluation, + catalog: &CurationProjectionCatalog<'_>, + config: &TmdbCurationConfig, + keys: &[CurationSelectorKey], +) -> Vec { + assert_eq!(keys.len(), config.trending.len()); + let mut groups = Vec::new(); + for (selector, key) in config.trending.iter().zip(keys) { + if !selector.create_xtream_category { + continue; + } + let Some(name) = selector.category_name.as_deref().filter(|name| !name.trim().is_empty()) else { + continue; + }; + let spec = CurationCategorySpec { + name, + selector: selector_spec(selector.kind), + projection_identity: ProjectionIdentityStrategy::LegacyCategoryScoped { + namespace: LEGACY_CATEGORY_NAMESPACE, + }, + }; + groups.extend(project_memberships( + evaluation.memberships.iter().filter(|membership| membership.selector_key == *key), + catalog, + &spec, + )); + } + groups +} diff --git a/backend/curation/src/tmdb/model.rs b/backend/curation/src/tmdb/model.rs new file mode 100644 index 000000000..9d0c0cd55 --- /dev/null +++ b/backend/curation/src/tmdb/model.rs @@ -0,0 +1,146 @@ +use crate::kernel::{CuratedMediaReference, CurationMediaKind}; +use serde::{Deserialize, Deserializer}; +use shared::model::TmdbTrendingKind; +use std::num::NonZeroU32; + +#[derive(Deserialize)] +struct TrendingPage { + page: u64, + total_pages: u64, + total_results: u64, + results: Vec, +} + +#[derive(Deserialize)] +struct TrendingItem { + id: NonZeroU32, + #[serde(default, deserialize_with = "present_kind")] + media_type: Option, + title: Option, + name: Option, + release_date: Option, + first_air_date: Option, +} + +fn present_kind<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + TmdbTrendingKind::deserialize(deserializer).map(Some) +} + +pub(super) struct ValidatedPage { + /// All rows, including duplicates and the suffix beyond the selection limit. + pub rows: Vec, + pub last: bool, +} + +pub(super) fn translate_page( + body: &[u8], + kind: TmdbTrendingKind, + requested_page: u64, + preceding_rows: u32, +) -> Result { + let page: TrendingPage = serde_json::from_slice(body).map_err(|_| ())?; + if requested_page == 0 + || page.page != requested_page + || page.total_results < u64::try_from(page.results.len()).map_err(|_| ())? + { + return Err(()); + } + if page.results.is_empty() { + return if requested_page == 1 && page.total_results == 0 && page.total_pages <= 1 { + Ok(ValidatedPage { rows: Vec::new(), last: true }) + } else { + Err(()) + }; + } + if page.total_pages < requested_page { + return Err(()); + } + let mut rows = Vec::with_capacity(page.results.len()); + for (index, item) in page.results.into_iter().enumerate() { + // Validate before deduplication or selection: even an unselected suffix is authoritative wire data. + if item.media_type.is_some_and(|media_type| media_type != kind) { + return Err(()); + } + let rank = preceding_rows + .checked_add(u32::try_from(index).map_err(|_| ())?) + .and_then(|rank| rank.checked_add(1)) + .ok_or(())?; + let (media_kind, title, date) = match kind { + TmdbTrendingKind::Movie => (CurationMediaKind::Movie, item.title, item.release_date), + TmdbTrendingKind::Tv => (CurationMediaKind::Series, item.name, item.first_air_date), + }; + let year = date + .as_deref() + .and_then(|date| date.split('-').next()) + .and_then(|year| year.parse::().ok()) + .filter(|year| (1900..=2100).contains(year)); + rows.push(CuratedMediaReference::new( + media_kind, + title.unwrap_or_default(), + year, + Some(item.id.get()), + Some(rank), + )); + } + Ok(ValidatedPage { rows, last: page.page == page.total_pages }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + fn translate(results: Value, kind: TmdbTrendingKind, preceding: u32) -> Result { + translate_page( + &serde_json::to_vec(&json!({"page":2,"total_pages":3,"total_results":100,"results":results})).unwrap(), + kind, + 2, + preceding, + ) + } + + #[test] + fn tmdb_wire_translation_preserves_every_row_and_cumulative_rank() { + let page = translate( + json!([{"id":7,"title":"First","release_date":"2024-01-02"},{"id":7,"title":"Duplicate"},{"id":u32::MAX}]), + TmdbTrendingKind::Movie, + 3, + ) + .unwrap(); + assert!(!page.last); + assert_eq!(page.rows.len(), 3); + assert_eq!(page.rows[0].title, "First"); + assert_eq!(page.rows[0].year, Some(2024)); + assert_eq!(page.rows[2].rank, Some(6)); + assert_eq!(page.rows[2].tmdb_id, Some(u32::MAX)); + let page = translate( + json!([{"id":7,"media_type":"tv","name":"Show","first_air_date":"unknown"},{"id":8,"name":null}]), + TmdbTrendingKind::Tv, + 0, + ) + .unwrap(); + assert_eq!(page.rows[0].kind, CurationMediaKind::Series); + assert_eq!(page.rows[0].title, "Show"); + assert_eq!(page.rows[0].year, None); + assert_eq!(page.rows[1].title, ""); + assert!(translate(json!([{"id":7}]), TmdbTrendingKind::Movie, u32::MAX).is_err()); + } + + #[test] + fn tmdb_wire_invalid_duplicate_or_suffix_is_not_discarded() { + for bad in [ + json!({}), + json!({"id":0}), + json!({"id":-1}), + json!({"id":4294967296_u64}), + json!({"id":"7"}), + json!({"id":7.0}), + json!({"id":7,"media_type":"tv"}), + json!({"id":7,"media_type":null}), + json!({"id":7,"title":42}), + json!({"id":7,"release_date":false}), + ] { + assert!(translate(json!([{"id":7},bad]), TmdbTrendingKind::Movie, 0).is_err()); + } + } +} diff --git a/backend/curation/src/trakt/mod.rs b/backend/curation/src/trakt/mod.rs index 7008ee612..6d1a10b5a 100644 --- a/backend/curation/src/trakt/mod.rs +++ b/backend/curation/src/trakt/mod.rs @@ -4,15 +4,14 @@ mod model; use crate::kernel::{ evaluate_selector, project_memberships, CuratedMediaReference, CurationCategorySpec, CurationEvaluation, - CurationFailure, CurationIncompleteReason, CurationMatchPolicy, CurationMediaScope, CurationProjectionCatalog, - CurationRunOutcome, CurationSelectorKey, CurationSelectorSpec, CurationSelectorSummary, CurationUnavailableReason, - ProjectionIdentityStrategy, SelectorOutcome, + CurationIncompleteReason, CurationMatchPolicy, CurationMediaScope, CurationProjectionCatalog, CurationRunOutcome, + CurationSelectorKey, CurationSelectorSpec, CurationUnavailableReason, ProjectionIdentityStrategy, SelectorOutcome, }; use client::{TraktClient, TraktFetchFailureKind}; use log::{debug, info, warn}; use model::TraktListItem; use shared::model::{PlaylistGroup, TraktContentType}; -use tuliprox_core::model::{TraktChartConfig, TraktConfig, TraktListConfig}; +use tuliprox_core::model::{CurationConfig, TraktChartConfig, TraktConfig, TraktListConfig, TraktSourceConfig}; // Compatibility policy for the current Xtream projection. This namespace is // deliberately supplied by the adapter rather than treated as canonical media identity. @@ -30,20 +29,26 @@ pub async fn evaluate_trakt_curation( if !trakt_config.enabled || (trakt_config.lists.is_empty() && trakt_config.charts.is_empty()) { return CurationRunOutcome::NotConfigured; } + crate::evaluate_curation(http_client, None, playlist, target_name, &CurationConfig::from(trakt_config)).await +} +pub(crate) async fn evaluate_selectors( + http_client: &reqwest::Client, + playlist: &[PlaylistGroup], + target_name: &str, + trakt_config: &TraktSourceConfig, + keys: &[CurationSelectorKey], +) -> Vec { let selector_count = trakt_config.lists.len() + trakt_config.charts.len(); - let processor = match TraktCategoriesProcessor::new(http_client, trakt_config) { + assert_eq!(selector_count, keys.len()); + let processor = match TraktClient::new(http_client.clone(), trakt_config.api.clone()) { Ok(processor) => processor, Err(error) => { warn!("Trakt curation is unavailable for target '{target_name}': {}", error.message()); - return CurationRunOutcome::Failed(CurationFailure { - selector_outcomes: (0..selector_count) - .map(|ordinal| SelectorOutcome::Unavailable { - key: selector_key(ordinal), - reason: CurationUnavailableReason::Configuration, - }) - .collect(), - }); + return keys + .iter() + .map(|key| SelectorOutcome::Unavailable { key: *key, reason: CurationUnavailableReason::Configuration }) + .collect(); } }; @@ -55,9 +60,9 @@ pub async fn evaluate_trakt_curation( let mut selector_outcomes = Vec::with_capacity(selector_count); for (ordinal, list_config) in trakt_config.lists.iter().enumerate() { - let key = selector_key(ordinal); + let key = keys[ordinal]; let source_label = format!("{}:{}", list_config.user, list_config.list_slug); - let outcome = match processor.client.get_list_items(list_config).await { + let outcome = match processor.get_list_items(list_config).await { Ok(items) => { debug!("Evaluating Trakt list {source_label} with {} items", items.len()); let references = translate_items(items); @@ -73,9 +78,9 @@ pub async fn evaluate_trakt_curation( for (chart_index, chart_config) in trakt_config.charts.iter().enumerate() { let ordinal = trakt_config.lists.len() + chart_index; - let key = selector_key(ordinal); + let key = keys[ordinal]; let source_label = format!("{}:{}", chart_config.kind, chart_config.chart); - let outcome = match processor.client.get_chart_items(chart_config).await { + let outcome = match processor.get_chart_items(chart_config).await { Ok(items) => { debug!("Evaluating Trakt chart {source_label} with {} items", items.len()); let references = translate_items(items); @@ -89,11 +94,9 @@ pub async fn evaluate_trakt_curation( selector_outcomes.push(outcome); } - complete_evaluation(selector_outcomes) + selector_outcomes } -const fn selector_key(ordinal: usize) -> CurationSelectorKey { CurationSelectorKey(ordinal) } - fn failed_selector_outcome(key: CurationSelectorKey, failure: TraktFetchFailureKind) -> SelectorOutcome { match failure { TraktFetchFailureKind::Interrupted => { @@ -114,26 +117,35 @@ pub fn project_trakt_categories( playlist: &[PlaylistGroup], trakt_config: &TraktConfig, ) -> Vec { + crate::project_curation_categories(evaluation, playlist, &CurationConfig::from(trakt_config)) +} + +pub(crate) fn project_categories( + evaluation: &CurationEvaluation, + projection_catalog: &CurationProjectionCatalog<'_>, + trakt_config: &TraktSourceConfig, + keys: &[CurationSelectorKey], +) -> Vec { + assert_eq!(keys.len(), trakt_config.lists.len() + trakt_config.charts.len()); let mut categories = Vec::new(); - let projection_catalog = CurationProjectionCatalog::new(playlist); for (ordinal, list_config) in trakt_config.lists.iter().enumerate() { append_selector_projection( - selector_key(ordinal), + keys[ordinal], list_config.category_name.as_deref(), list_config.create_xtream_category, evaluation, - &projection_catalog, + projection_catalog, &list_category_spec(list_config), &mut categories, ); } for (chart_index, chart_config) in trakt_config.charts.iter().enumerate() { append_selector_projection( - selector_key(trakt_config.lists.len() + chart_index), + keys[trakt_config.lists.len() + chart_index], chart_config.category_name.as_deref(), chart_config.create_xtream_category, evaluation, - &projection_catalog, + projection_catalog, &chart_category_spec(chart_config), &mut categories, ); @@ -157,34 +169,6 @@ fn append_selector_projection( categories.extend(project_memberships(memberships, projection_catalog, specification)); } -fn complete_evaluation(selector_outcomes: Vec) -> CurationRunOutcome { - if selector_outcomes.iter().any(|outcome| !matches!(outcome, SelectorOutcome::Complete { .. })) { - return CurationRunOutcome::Failed(CurationFailure { selector_outcomes }); - } - - let mut selectors = Vec::with_capacity(selector_outcomes.len()); - let mut memberships = Vec::new(); - for outcome in selector_outcomes { - let SelectorOutcome::Complete { key, reference_count, memberships: mut selector_memberships } = outcome else { - unreachable!("all selector outcomes were checked as complete") - }; - selectors.push(CurationSelectorSummary { key, reference_count, membership_count: selector_memberships.len() }); - memberships.append(&mut selector_memberships); - } - CurationRunOutcome::Complete(CurationEvaluation { selectors, memberships }) -} - -struct TraktCategoriesProcessor { - client: TraktClient, -} - -impl TraktCategoriesProcessor { - fn new(http_client: &reqwest::Client, trakt_config: &TraktConfig) -> Result { - let client = TraktClient::new(http_client.clone(), trakt_config.api.clone())?; - Ok(Self { client }) - } -} - fn translate_items(items: Vec) -> Vec { items.into_iter().filter_map(TraktListItem::into_curated_reference).collect() } @@ -243,11 +227,13 @@ fn category_spec( }, } } - #[cfg(test)] mod tests { use super::*; - use crate::trakt::model::{TraktIds, TraktMovie, TraktShow}; + use crate::{ + coordinator::complete_evaluation, + trakt::model::{TraktIds, TraktMovie, TraktShow}, + }; use shared::{ model::{ EpisodeStreamProperties, FieldGet, HeaderField, PlaylistItem, PlaylistItemHeader, PlaylistItemType, diff --git a/backend/processing/Cargo.toml b/backend/processing/Cargo.toml index 8a19ab37a..0e5fb440d 100644 --- a/backend/processing/Cargo.toml +++ b/backend/processing/Cargo.toml @@ -46,5 +46,7 @@ tokio-util = { version = "0.7.19", features = ["io", "io-util"] } [dev-dependencies] rand = "0.9.5" +rcgen = "0.14.9" rmp-serde = "1.3.1" +tokio-rustls = "0.26.4" tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/backend/processing/src/processor/playlist/mod.rs b/backend/processing/src/processor/playlist/mod.rs index 11b9f0807..31a9239cb 100644 --- a/backend/processing/src/processor/playlist/mod.rs +++ b/backend/processing/src/processor/playlist/mod.rs @@ -49,14 +49,14 @@ use tokio::{ use tuliprox_core::{ model::{ is_valid, retain_filtered_playlist, AppConfig, ClusterForceUpdate, ClusterUpdateRejection, CompiledMapping, - ConfigFavourites, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, Epg, - FilterOutcome, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, ReverseProxyDisabledHeaderConfig, - TraktConfig, TransformStage, UpdateGuard, UpdateTask, + ConfigFavourites, ConfigInput, ConfigInputFlags, ConfigInputOptions, ConfigRename, ConfigTarget, + CurationConfig, Epg, FilterOutcome, MappingProgram, ProcessTargets, ProviderIdType, ResolveReason, + ReverseProxyDisabledHeaderConfig, TransformStage, UpdateGuard, UpdateTask, }, utils::{debug_if_enabled, log_memory_snapshot, trace_if_enabled, StepMeasure, StepMeasureCallback}, }; use tuliprox_curation::{ - evaluate_trakt_curation, project_trakt_categories, CurationEvaluation, CurationFailure, CurationRunOutcome, + evaluate_curation, project_curation_categories, CurationEvaluation, CurationFailure, CurationRunOutcome, SelectorOutcome, }; use tuliprox_iptv::{ diff --git a/backend/processing/src/processor/playlist/target.rs b/backend/processing/src/processor/playlist/target.rs index 43eb6e745..87921e56f 100644 --- a/backend/processing/src/processor/playlist/target.rs +++ b/backend/processing/src/processor/playlist/target.rs @@ -463,9 +463,49 @@ fn finalize_playlist_view( playlist } +// Keep construction and its diagnostic seam private to the processing boundary. +// Only fixed phase labels can reach the reporter, never dependency error details. +pub(super) fn build_target_tmdb_client( + target: &ConfigTarget, + configure: impl FnOnce() -> Result, + report: impl FnOnce(&'static str), +) -> Option { + let needs_tmdb = target.curation.as_ref().is_some_and(|config| { + config.enabled && config.tmdb.as_ref().is_some_and(|source| source.enabled && !source.trending.is_empty()) + }); + if !needs_tmdb { + return None; + } + let Ok(builder) = configure() else { + report("phase=profile_configuration"); + return None; + }; + if let Ok(client) = builder.build() { + Some(client) + } else { + report("phase=client_build"); + None + } +} pub(crate) async fn finalize_prepared_target( ctx: Arc>, prepared: PreparedTarget, +) -> (Result<(), Vec>, Vec) { + // No generic-client fallback: unavailable transport participates in the same admission gate. + let tmdb_client = build_target_tmdb_client( + &prepared.target, + || tuliprox_core::utils::network::request::create_tmdb_client(&ctx.config), + |phase| { + warn!("TMDB discovery client unavailable: target_id={} run_id={} {phase}", prepared.target.id, ctx.run_id); + }, + ); + finalize_prepared_target_with_tmdb(ctx, prepared, tmdb_client.as_ref()).await +} + +pub(super) async fn finalize_prepared_target_with_tmdb( + ctx: Arc>, + prepared: PreparedTarget, + tmdb_client: Option<&reqwest::Client>, ) -> (Result<(), Vec>, Vec) { let target = &prepared.target; let mut new_playlist = prepared.playlist; @@ -488,7 +528,7 @@ pub(crate) async fn finalize_prepared_target views, Err(error) => { step.stop("Curation failed; skipping persist to preserve finalized artifacts"); @@ -864,20 +904,24 @@ pub(crate) struct TargetPlaylistViews { pub(crate) fn build_curated_playlist_views( playlist: Vec, evaluation: &CurationEvaluation, - trakt_config: &TraktConfig, + config: &CurationConfig, appearance_filter_configured: bool, + has_xtream_output: bool, ) -> TargetPlaylistViews { - let curated_catalog = trakt_config.catalog_selection.is_curated(); - let mut categories = project_trakt_categories(evaluation, &playlist, trakt_config); + let curated_catalog = config.catalog_selection.is_curated(); + let categories = has_xtream_output.then(|| project_curation_categories(evaluation, &playlist, config)); let base = select_target_catalog(playlist, evaluation, curated_catalog); - let mut xtream = if trakt_config.include_xtream_base_categories { base.clone() } else { live_only(&base) }; - xtream.append(&mut categories); + let xtream = categories.map(|mut categories| { + let mut view = if config.include_xtream_base_categories { base.clone() } else { live_only(&base) }; + view.append(&mut categories); + view + }); TargetPlaylistViews { base, - xtream: Some(xtream), + xtream, publication_plan: PlaylistPublicationPlan::complete_curation_with_filter( curated_catalog, - !trakt_config.include_xtream_base_categories, + has_xtream_output && !config.include_xtream_base_categories, appearance_filter_configured, ), } @@ -885,10 +929,11 @@ pub(crate) fn build_curated_playlist_views( pub(crate) async fn prepare_target_playlist_views( client: &reqwest::Client, + tmdb_client: Option<&reqwest::Client>, target: &ConfigTarget, playlist: Vec, ) -> Result { - let Some(trakt_config) = target.get_xtream_output().and_then(|xtream| xtream.trakt.as_ref()) else { + let Some(config) = target.effective_curation() else { return Ok(TargetPlaylistViews { base: playlist, xtream: None, @@ -896,7 +941,17 @@ pub(crate) async fn prepare_target_playlist_views( }); }; - match evaluate_trakt_curation(client, &playlist, &target.name, trakt_config).await { + let outcome = evaluate_curation(client, tmdb_client, &playlist, &target.name, &config).await; + curation_playlist_views(target, playlist, &config, outcome) +} + +pub(super) fn curation_playlist_views( + target: &ConfigTarget, + playlist: Vec, + config: &CurationConfig, + outcome: CurationRunOutcome, +) -> Result { + match outcome { CurationRunOutcome::NotConfigured => Ok(TargetPlaylistViews { base: playlist, xtream: None, @@ -904,8 +959,13 @@ pub(crate) async fn prepare_target_playlist_views( }), CurationRunOutcome::Failed(failure) => Err(curation_failure_error(&target.name, &failure)), CurationRunOutcome::Complete(evaluation) => { - let views = - build_curated_playlist_views(playlist, &evaluation, trakt_config, target.filter.persist.is_some()); + let views = build_curated_playlist_views( + playlist, + &evaluation, + config, + target.filter.persist.is_some(), + target.get_xtream_output().is_some(), + ); info!( "Target '{}' curation completed with {} memberships and {} Xtream groups", target.name, @@ -916,7 +976,6 @@ pub(crate) async fn prepare_target_playlist_views( } } } - fn curation_failure_error(target_name: &str, failure: &CurationFailure) -> TuliproxError { let mut complete = 0usize; let mut incomplete = 0usize; diff --git a/backend/processing/src/processor/playlist/tests.rs b/backend/processing/src/processor/playlist/tests.rs index ca07cf72f..fb1a2fc71 100644 --- a/backend/processing/src/processor/playlist/tests.rs +++ b/backend/processing/src/processor/playlist/tests.rs @@ -16,6 +16,8 @@ use tuliprox_curation::{ CurationEvaluation, CurationMediaKind, CurationMembership, CurationSelectorKey, CurationSelectorSummary, }; +mod curation_publication; + fn apply_staged_overlay_groups( provider: &ConfigInput, clusters: ClusterFlags, @@ -2709,7 +2711,7 @@ fn persist_filter_can_select_a_base_group() { async fn trakt_target_curation_is_a_noop_without_xtream_configuration() { let target = ConfigTarget::from(&ConfigTargetDto::default()); - let views = prepare_target_playlist_views(&reqwest::Client::new(), &target, Vec::new()) + let views = prepare_target_playlist_views(&reqwest::Client::new(), None, &target, Vec::new()) .await .expect("unconfigured curation should not fail"); @@ -2752,7 +2754,7 @@ async fn unavailable_required_selector_returns_target_failure_instead_of_base_fa xtream_cluster: XtreamCluster::Video, }]; - let error = prepare_target_playlist_views(&reqwest::Client::new(), &target, base) + let error = prepare_target_playlist_views(&reqwest::Client::new(), None, &target, base) .await .expect_err("missing credentials must stop target publication"); @@ -2778,7 +2780,7 @@ mod curation_effect_gate { utils::FileLockManager, }; - fn app_config(storage_dir: &Path) -> Arc { + pub(super) fn app_config(storage_dir: &Path) -> Arc { Arc::new(AppConfig { config: Arc::new(ArcSwap::from_pointee(Config { storage_dir: storage_dir.to_string_lossy().into_owned(), @@ -2808,7 +2810,7 @@ mod curation_effect_gate { }) } - fn processing_context( + pub(super) fn processing_context( app_config: Arc, playlist_state: Option>, ) -> PlaylistProcessingContext { @@ -2844,7 +2846,7 @@ mod curation_effect_gate { TargetPlaylistPersistOptions { publication_plan, ..TargetPlaylistPersistOptions::default() } } - fn file_snapshot(root: &Path) -> BTreeMap> { + pub(super) fn file_snapshot(root: &Path) -> BTreeMap> { fn collect(root: &Path, path: &Path, snapshot: &mut BTreeMap>) { let Ok(entries) = std::fs::read_dir(path) else { return }; for entry in entries.flatten() { @@ -2892,10 +2894,35 @@ mod curation_effect_gate { #[tokio::test] async fn unavailable_curation_retains_seeded_artifacts_cache_and_watch_state() { + assert_curation_retains_seeded_state(None).await; + } + + #[tokio::test] + async fn tmdb_and_mixed_failures_preserve_files_ids_cache_and_watches_under_full_and_curated() { + for policy in ["full", "curated"] { + for mixed in [false, true] { + let (url, server) = empty_trakt_server().await; + let mut value = serde_json::json!({"catalog_selection": policy, "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 100, "category_name": "TMDB"}]}}); + if mixed { + value["trakt"] = serde_json::json!({"api": {"api_key": "test-client", "url": url}, "charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt"}]}); + } + let mut dto: shared::model::CurationConfigDto = serde_json::from_value(value).unwrap(); + dto.prepare(&[TargetOutputDto::Xtream(XtreamTargetOutputDto::default())]).unwrap(); + assert_curation_retains_seeded_state(Some(dto)).await; + if mixed { + server.await.unwrap(); + } else { + server.abort(); + } + } + } + } + + async fn assert_curation_retains_seeded_state(canonical: Option) { let directory = tempdir().expect("tempdir"); let app_config = app_config(directory.path()); let playlist_state = Arc::new(PlaylistStorageState::new()); - let target = ConfigTarget::from(&ConfigTargetDto { + let mut target = ConfigTarget::from(&ConfigTargetDto { name: "curation-effect-gate".to_string(), output: vec![ TargetOutputDto::Xtream(XtreamTargetOutputDto { @@ -2922,6 +2949,12 @@ mod curation_effect_gate { use_memory_cache: true, ..ConfigTargetDto::default() }); + if let Some(canonical) = canonical { + target.curation = Some(CurationConfig::from(&canonical)); + if let tuliprox_core::model::TargetOutput::Xtream(output) = &mut target.output[0] { + output.trakt = None; + } + } let mut seeded = vec![PlaylistGroup { id: 1, title: "Movies".intern(), @@ -3064,6 +3097,68 @@ mod curation_effect_gate { } } +#[test] +fn tmdb_complete_target_selection_preserves_live_and_keeps_xtream_aliases_out_of_normal_outputs() { + for has_xtream in [false, true] { + let mut value = serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 100, "create_xtream_category": has_xtream, "category_name": "TMDB"}]}}}); + if has_xtream { + value["output"].as_array_mut().unwrap().push(serde_json::json!({"type": "xtream"})); + } + let mut dto: ConfigTargetDto = serde_json::from_value(value).unwrap(); + dto.prepare(1, None, None).unwrap(); + let target = ConfigTarget::from(&dto); + let id = shared::utils::hash_string("selected"); + let live_id = shared::utils::hash_string("live"); + let playlist = vec![ + PlaylistGroup { + id: 1, + title: "Live".intern(), + xtream_cluster: XtreamCluster::Live, + channels: vec![catalog_test_item("Live", live_id, PlaylistItemType::Live, XtreamCluster::Live, None)], + }, + PlaylistGroup { + id: 2, + title: "Movies".intern(), + xtream_cluster: XtreamCluster::Video, + channels: vec![ + catalog_test_item("Selected", id, PlaylistItemType::Video, XtreamCluster::Video, None), + catalog_test_item( + "Other", + shared::utils::hash_string("other"), + PlaylistItemType::Video, + XtreamCluster::Video, + None, + ), + ], + }, + ]; + let config = target.effective_curation().unwrap(); + let outcome = CurationRunOutcome::Complete(complete_catalog_evaluation(vec![catalog_membership( + id, + CurationMediaKind::Movie, + 0, + )])); + let views = target::curation_playlist_views(&target, playlist.clone(), &config, outcome).unwrap(); + assert_eq!( + views.base.iter().flat_map(|g| &g.channels).map(|i| i.header.uuid).collect::>(), + [live_id, id] + ); + if has_xtream { + let groups = views.xtream.unwrap(); + let projected = groups.iter().find(|g| g.title.as_ref() == "TMDB").unwrap(); + assert_ne!(projected.channels[0].header.uuid, id); + } else { + assert!(views.xtream.is_none()); + } + let empty = CurationRunOutcome::Complete(complete_catalog_evaluation(Vec::new())); + let empty_views = target::curation_playlist_views(&target, playlist, &config, empty).unwrap(); + assert_eq!(empty_views.base.len(), 1); + assert_eq!(empty_views.base[0].xtream_cluster, XtreamCluster::Live); + assert_ne!(empty_views.publication_plan, PlaylistPublicationPlan::Ordinary); + } +} + fn catalog_test_item( title: &str, uuid: UUIDType, @@ -3245,7 +3340,8 @@ fn xtream_base_and_selector_category_projection_are_independent() { }; let config = TraktConfig::from(&dto); - let views = build_curated_playlist_views(playlist.clone(), &evaluation, &config, false); + let views = + build_curated_playlist_views(playlist.clone(), &evaluation, &CurationConfig::from(&config), false, true); assert_eq!(views.base.len(), 1); assert_eq!(views.base[0].channels.len(), 1); @@ -3264,7 +3360,13 @@ fn xtream_base_and_selector_category_projection_are_independent() { compatible_dto.include_xtream_base_categories = true; compatible_dto.lists[0].create_xtream_category = false; compatible_dto.lists[0].category_name = None; - let compatible = build_curated_playlist_views(playlist, &evaluation, &TraktConfig::from(&compatible_dto), false); + let compatible = build_curated_playlist_views( + playlist, + &evaluation, + &CurationConfig::from(&TraktConfig::from(&compatible_dto)), + false, + true, + ); assert_eq!(compatible.base[0].channels.len(), 2); assert_eq!(compatible.xtream.expect("Xtream view").len(), 1, "selection-only selector creates no category"); } @@ -3313,7 +3415,8 @@ fn curation_policy_truth_table_covers_a_through_h() { charts: Vec::new(), }); - let views = build_curated_playlist_views(playlist.clone(), &evaluation, &config, false); + let views = + build_curated_playlist_views(playlist.clone(), &evaluation, &CurationConfig::from(&config), false, true); let base_items = views.base.iter().flat_map(|group| &group.channels).collect::>(); let expected_base_count = if catalog_selection == TraktCatalogSelection::Full { 2 } else { 1 }; assert_eq!(base_items.len(), expected_base_count, "case {case} selected catalog"); @@ -3402,7 +3505,7 @@ fn large_catalog_projection_smoke_keeps_two_explicit_views_bounded() { ..TraktConfigDto::default() }); - let views = build_curated_playlist_views(playlist, &evaluation, &config, false); + let views = build_curated_playlist_views(playlist, &evaluation, &CurationConfig::from(&config), false, true); assert_eq!(views.base[0].channels.len(), CATALOG_SIZE); let xtream = views.xtream.expect("Xtream view"); @@ -3881,6 +3984,7 @@ mod quality_rejection_fallback { fn xtream_target(use_memory_cache: bool) -> Arc { Arc::new(ConfigTarget { + curation: None, id: 1, enabled: true, name: "quality-target".to_string(), diff --git a/backend/processing/src/processor/playlist/tests/curation_publication.rs b/backend/processing/src/processor/playlist/tests/curation_publication.rs new file mode 100644 index 000000000..89f8d004c --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/curation_publication.rs @@ -0,0 +1,516 @@ +//! Real HTTPS adapters -> eligible catalog -> target finalization -> disk/cache/watch publication. +//! DNS overrides and a fixture-only CA keep the production TMDB origin and certificate validation intact. +mod client_diagnostics; +mod item_limit; +mod tls_profile; +use super::{ + curation_effect_gate::{app_config, file_snapshot, processing_context}, + *, +}; +use serde_json::json; +use shared::{ + model::{SeriesStreamProperties, StreamProperties, VideoStreamProperties}, + utils::hash_string, +}; +use std::{collections::BTreeMap, sync::Mutex as StdMutex, time::Duration}; +use tempfile::{tempdir, TempDir}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + task::JoinHandle, +}; +use tokio_rustls::{rustls, TlsAcceptor}; +use tuliprox_repository::{load_m3u_target_storage, load_xtream_target_storage}; + +const MOVIES: &str = "/3/trending/movie/week?language=en-US&page=1"; +const MOVIES_2: &str = "/3/trending/movie/week?language=en-US&page=2"; +const TV: &str = "/3/trending/tv/day?language=en-US&page=1"; +const TRAKT: &str = "/movies/popular?page=1&limit=100"; +const MOVIE_PAGE: &str = + r#"{"page":1,"total_pages":9,"total_results":99,"results":[{"id":8,"media_type":"movie"},{"id":8}]}"#; +const MOVIE_PAGE_2: &str = r#"{"page":2,"total_pages":2,"total_results":2,"results":[{"id":8},{"id":7}]}"#; +const TV_PAGE: &str = r#"{"page":1,"total_pages":1,"total_results":1,"results":[{"id":7,"media_type":"tv"}]}"#; +const TRAKT_PAGE: &str = r#"[{"title":"Unselected","ids":{"tmdb":99,"trakt":1,"slug":"unselected"}},{"title":"First","ids":{"tmdb":7,"trakt":2,"slug":"first"}}]"#; +const EMPTY: &str = r#"{"page":1,"total_pages":0,"total_results":0,"results":[]}"#; + +struct DiscoveryServer { + client: reqwest::Client, + tmdb_client: reqwest::Client, + certificate: reqwest::Certificate, + address: std::net::SocketAddr, + replies: Arc>>>, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl DiscoveryServer { + async fn start() -> Self { + let cert = + rcgen::generate_simple_self_signed(vec!["api.themoviedb.org".into(), "api.trakt.tv".into()]).unwrap(); + let tls = rustls::ServerConfig::builder_with_provider(Arc::new(rustls::crypto::aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert( + vec![cert.cert.der().clone()], + rustls::pki_types::PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()).into(), + ) + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let client = reqwest::Client::builder() + .no_proxy() + .add_root_certificate(reqwest::Certificate::from_der(cert.cert.der()).unwrap()) + .resolve("api.themoviedb.org", address) + .resolve("api.trakt.tv", address) + .timeout(Duration::from_secs(3)) + .build() + .unwrap(); + let profile_directory = tempdir().unwrap(); + let tmdb_client = + tuliprox_core::utils::network::request::create_tmdb_client(&app_config(profile_directory.path())) + .unwrap() + .no_proxy() + .add_root_certificate(reqwest::Certificate::from_der(cert.cert.der()).unwrap()) + .resolve("api.themoviedb.org", address) + .build() + .unwrap(); + let replies = Arc::new(StdMutex::new(BTreeMap::from([ + (MOVIES.into(), response(200, MOVIE_PAGE)), + (MOVIES_2.into(), response(200, MOVIE_PAGE_2)), + (TV.into(), response(200, TV_PAGE)), + (TRAKT.into(), response(200, TRAKT_PAGE)), + ]))); + let requests = Arc::new(StdMutex::new(Vec::new())); + let responses = Arc::clone(&replies); + let recorded = Arc::clone(&requests); + let task = tokio::spawn(async move { + let acceptor = TlsAcceptor::from(Arc::new(tls)); + loop { + let (tcp, _) = listener.accept().await.unwrap(); + let Ok(mut stream) = acceptor.accept(tcp).await else { continue }; + let mut bytes = Vec::new(); + while !bytes.windows(4).any(|w| w == b"\r\n\r\n") { + let mut chunk = [0; 1024]; + let count = stream.read(&mut chunk).await.unwrap(); + assert!(count > 0 && bytes.len() < 16_384, "bounded fixture request headers"); + bytes.extend_from_slice(&chunk[..count]); + } + let request = String::from_utf8(bytes).unwrap(); + let path = request.split_whitespace().nth(1).unwrap(); + let response = responses.lock().unwrap().get(path).cloned().unwrap_or_else(|| response(404, "{}")); + recorded.lock().unwrap().push(request); + let _ = stream.write_all(&response).await; + let _ = stream.shutdown().await; + } + }); + Self { + client, + tmdb_client, + certificate: reqwest::Certificate::from_der(cert.cert.der()).unwrap(), + address, + replies, + requests, + task, + } + } + + fn reply(&self, path: &str, status: u16, body: &str) { self.reply_raw(path, response(status, body)); } + + fn reply_raw(&self, path: &str, body: Vec) { self.replies.lock().unwrap().insert(path.into(), body); } +} + +fn response(status: u16, body: &str) -> Vec { + format!("HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).into_bytes() +} + +impl Drop for DiscoveryServer { + fn drop(&mut self) { self.task.abort(); } +} + +fn catalog() -> Vec { + let mut groups = Vec::new(); + for (cluster, rows) in [ + (XtreamCluster::Live, vec![(101, "Live", 0)]), + ( + XtreamCluster::Video, + vec![(201, "First", 7), (202, "Local alias", 7), (203, "Second", 8), (204, "Unselected", 99)], + ), + (XtreamCluster::Series, vec![(301, "Show", 7), (302, "Other show", 9)]), + ] { + let group = match cluster { + XtreamCluster::Live => "Live", + XtreamCluster::Video => "Movies", + XtreamCluster::Series => "Series", + }; + let mut channels = Vec::new(); + for (id, title, tmdb) in rows { + let properties = match cluster { + XtreamCluster::Live => None, + XtreamCluster::Video => Some(StreamProperties::Video(Box::new(VideoStreamProperties { + tmdb: Some(tmdb), + ..Default::default() + }))), + XtreamCluster::Series => Some(StreamProperties::Series(Box::new(SeriesStreamProperties { + tmdb: Some(tmdb), + ..Default::default() + }))), + }; + let item_type = match cluster { + XtreamCluster::Live => PlaylistItemType::Live, + XtreamCluster::Video => PlaylistItemType::Video, + XtreamCluster::Series => PlaylistItemType::SeriesInfo, + }; + let header = PlaylistItemHeader { + id: id.to_string().intern(), + uuid: hash_string(title), + name: title.intern(), + title: title.intern(), + group: group.intern(), + url: format!("http://media.invalid/{id}.mkv").intern(), + item_type, + xtream_cluster: cluster, + epg_channel_id: (cluster == XtreamCluster::Live).then(|| "fixture.live".intern()), + additional_properties: properties, + ..Default::default() + }; + channels.push(PlaylistItem { header }); + if cluster == XtreamCluster::Series { + let title_episode = format!("{title} S01E01"); + channels.push(PlaylistItem { + header: PlaylistItemHeader { + id: (id + 100).to_string().intern(), + uuid: hash_string(&title_episode), + name: title_episode.as_str().intern(), + title: title_episode.as_str().intern(), + group: group.intern(), + item_type: PlaylistItemType::Series, + parent_code: hash_string(title).to_string().intern(), + xtream_cluster: cluster, + url: format!("http://media.invalid/{}.mkv", id + 100).intern(), + additional_properties: Some(StreamProperties::Episode(Box::new( + serde_json::from_value(json!({ + "episode_id": id + 100, "episode": 1, "season": 1 + })) + .unwrap(), + ))), + ..Default::default() + }, + }); + } + } + groups.push(PlaylistGroup { + id: u32::try_from(groups.len() + 1).unwrap(), + title: group.intern(), + xtream_cluster: cluster, + channels, + }); + } + groups +} + +fn target(policy: &str, mixed: bool, xtream: bool) -> ConfigTargetDto { + let mut value = json!({"name":"publication", "watch":[".*"], "use_memory_cache":true, + "output":[{"type":"m3u","filename":"published.m3u"},{"type":"strm","directory":"strm","flat":true,"cleanup":true}], + "curation":{"catalog_selection":policy,"tmdb":{"api":{"access_token":"fixture-discovery-token"},"trending":[ + {"kind":"movie","time_window":"week","limit":100,"create_xtream_category":xtream,"category_name":"TMDB Movies"}, + {"kind":"tv","time_window":"day","limit":100,"create_xtream_category":xtream,"category_name":"TMDB TV"} + ]}}}); + if xtream { + value["output"].as_array_mut().unwrap().push(json!({"type":"xtream"})); + } + if mixed { + value["curation"]["trakt"] = json!({"api":{"api_key":"fixture-trakt-client"},"charts":[ + {"kind":"movies","chart":"popular","category_name":"Trakt Movies","tmdb_only":true} + ]}); + } + let mut dto: ConfigTargetDto = serde_json::from_value(value).unwrap(); + dto.prepare(1, None, None).unwrap(); + dto +} + +struct Publication { + directory: TempDir, + context: Arc>, + target: ConfigTarget, + tmdb_client: reqwest::Client, +} + +impl Publication { + fn new(server: &DiscoveryServer, dto: &ConfigTargetDto) -> Self { + let directory = tempdir().unwrap(); + let mut context = processing_context(app_config(directory.path()), Some(Arc::new(PlaylistStorageState::new()))); + context.client = server.client.clone(); + Self { + directory, + context: Arc::new(context), + target: ConfigTarget::from(dto), + tmdb_client: server.tmdb_client.clone(), + } + } + + async fn publish(&self) -> Result<(), Vec> { self.publish_catalog(catalog()).await } + + async fn publish_catalog(&self, playlist: Vec) -> Result<(), Vec> { + let mut channel = shared::model::EpgChannel::new("fixture.live".intern()); + channel.title = Some("Fixture Live".intern()); + channel.programmes.push(shared::model::EpgProgramme::new_all( + 2_000_000_000, + 2_000_003_600, + "fixture.live".intern(), + playlist.first().and_then(|group| group.channels.first()).map(|item| Arc::clone(&item.header.title)), + None, + None, + )); + let prepared = PreparedTarget { + target: self.target.clone(), + playlist, + epg: vec![tuliprox_core::model::Epg { + priority: 0, + logo_override: false, + attributes: None, + children: vec![Arc::new(channel)], + }], + processing: PipelineStats::default(), + accepted_empty_clusters: ClusterFlags::empty(), + library_empty: tuliprox_repository::LibraryEmptyPublication::None, + }; + let (result, errors) = tokio::time::timeout( + Duration::from_secs(5), + target::finalize_prepared_target_with_tmdb(Arc::clone(&self.context), prepared, Some(&self.tmdb_client)), + ) + .await + .expect("bounded publication"); + assert!(errors.is_empty(), "unexpected non-curation errors: {errors:?}"); + result + } + + async fn xtream_rows(&self) -> Vec { + let storage = load_xtream_target_storage(&self.context.config, &self.target).await.unwrap(); + storage + .live + .iter() + .chain(storage.vod.iter()) + .chain(storage.series.iter()) + .map(|(_, item)| item.clone()) + .collect() + } + + async fn cache_signature(&self) -> Vec { + let cache = self.context.playlist_state.as_ref().unwrap().data.read().await; + let Some(storage) = cache.get(&self.target.name) else { return Vec::new() }; + let mut values = vec![format!( + "target:{}:xtream={}:m3u={}:mapping={}", + self.target.name, + storage.xtream.is_some(), + storage.m3u.is_some(), + storage.id_mapping.is_some() + )]; + if let Some(mapping) = &storage.id_mapping { + values.extend(mapping.iter().map(|(key, record)| format!("mapping:{key:?}:{record:?}"))); + } + if let Some(xtream) = &storage.xtream { + values.extend( + xtream + .live + .iter() + .chain(xtream.vod.iter()) + .chain(xtream.series.iter()) + .map(|(_, item)| format!("{item:?}")), + ); + } + if let Some(m3u) = &storage.m3u { + values.extend(m3u.iter().map(|(_, item)| format!("{item:?}"))); + } + values.sort(); + values + } + + async fn identity_signature(&self) -> BTreeMap { + let cache = self.context.playlist_state.as_ref().unwrap().data.read().await; + cache[&self.target.name] + .id_mapping + .as_ref() + .unwrap() + .iter() + .map(|(_, record)| (record.uuid.to_string(), (record.virtual_id.get(), record.parent_virtual_id.get()))) + .collect() + } + + fn strm_contents(&self) -> Vec { + file_snapshot(&self.directory.path().join("strm")) + .into_iter() + .filter(|(path, _)| path.extension().is_some_and(|ext| ext == "strm")) + .map(|(_, body)| String::from_utf8(body).unwrap()) + .collect() + } +} + +#[tokio::test] +async fn tmdb_https_to_publication_preserves_rank_subjects_series_and_output_boundaries() { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target("curated", false, true)); + run.publish().await.unwrap(); + let rows = run.xtream_rows().await; + let mut projected = rows.iter().filter(|item| item.group.as_ref() == "TMDB Movies").collect::>(); + projected.sort_by_key(|item| item.source_ordinal); + assert_eq!( + projected.iter().map(|item| item.title.as_ref()).collect::>(), + ["Second", "First", "Local alias"] + ); + assert!(rows.iter().any(|item| item.group.as_ref() == "TMDB TV" && item.title.as_ref() == "Show")); + assert!(!rows.iter().any(|item| item.title.as_ref() == "Unselected" || item.title.starts_with("Other show"))); + let episode = rows + .iter() + .find(|item| item.group.as_ref() == "TMDB TV" && item.item_type == PlaylistItemType::Series) + .expect("projected episode closure"); + let root = rows + .iter() + .find(|item| item.group.as_ref() == "TMDB TV" && item.item_type == PlaylistItemType::SeriesInfo) + .unwrap(); + assert_eq!( + episode.parent_code.as_ref(), + hash_string(&format!("trakt-category:TMDB TV:{}", hash_string("Show"))).to_string() + ); + { + let cache = run.context.playlist_state.as_ref().unwrap().data.read().await; + let mapping = cache[&run.target.name].id_mapping.as_ref().unwrap(); + let record = + mapping.iter().map(|(_, record)| record).find(|record| record.virtual_id == episode.virtual_id).unwrap(); + assert_eq!(record.parent_virtual_id, root.virtual_id, "persisted episode points to the projected series"); + } + let m3u = load_m3u_target_storage(&run.context.config, &run.target).await.unwrap(); + assert!(m3u.iter().all(|(_, item)| !item.group.starts_with("TMDB"))); + let text = std::fs::read_to_string(run.directory.path().join("published.m3u")).unwrap(); + for name in ["Live", "First", "Second", "Local alias", "Show S01E01"] { + assert!(text.contains(name), "missing {name}"); + } + assert!(!text.contains("Unselected") && !text.contains("Other show") && !text.contains("TMDB")); + let strm = run.strm_contents(); + for id in [201, 202, 203, 401] { + assert_eq!(strm.iter().filter(|body| body.contains(&format!("/{id}.mkv"))).count(), 1); + } + assert!(!strm.iter().any(|body| body.contains("/204.mkv") || body.contains("/402.mkv"))); + let before = run.identity_signature().await; + run.publish().await.unwrap(); + assert_eq!(run.identity_signature().await, before, "repeat publication retains virtual IDs and aliases"); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 6, "two movie pages and one TV page per run, no per-item fetch"); + assert!(requests.iter().all(|r| r.to_lowercase().contains("authorization: bearer fixture-discovery-token"))); +} + +#[tokio::test] +async fn mixed_https_publication_unions_subjects_without_duplicating_base_outputs_under_both_policies() { + for policy in ["full", "curated"] { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target(policy, true, true)); + run.publish().await.unwrap(); + let rows = run.xtream_rows().await; + assert!(rows.iter().any(|item| item.group.as_ref() == "Trakt Movies" && item.title.as_ref() == "Unselected")); + assert!(rows.iter().any(|item| item.group.as_ref() == "TMDB Movies" && item.title.as_ref() == "First")); + assert_eq!(rows.iter().any(|item| item.title.as_ref() == "Other show"), policy == "full"); + let m3u = load_m3u_target_storage(&run.context.config, &run.target).await.unwrap(); + for name in ["First", "Local alias", "Unselected"] { + assert_eq!(m3u.iter().filter(|(_, item)| item.title.as_ref() == name).count(), 1, "union once: {name}"); + } + assert!(m3u.iter().all(|(_, item)| !item.group.starts_with("TMDB") && !item.group.starts_with("Trakt"))); + assert_eq!(server.requests.lock().unwrap().len(), 4); + } +} + +#[tokio::test] +async fn tmdb_https_selection_only_publishes_m3u_strm_without_an_xtream_output() { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target("curated", false, false)); + run.publish().await.unwrap(); + let text = std::fs::read_to_string(run.directory.path().join("published.m3u")).unwrap(); + assert!(text.contains("First") && text.contains("Show S01E01") && text.contains("Live")); + assert!(!text.contains("Unselected") && !text.contains("TMDB")); + assert!(!run.strm_contents().is_empty()); + let cache = run.context.playlist_state.as_ref().unwrap().data.read().await; + assert!(cache[&run.target.name].xtream.is_none()); + assert_eq!(server.requests.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn tmdb_https_empty_and_no_match_clear_published_vod_series_but_keep_live() { + for body in + [EMPTY, r#"{"page":1,"total_pages":1,"total_results":1,"results":[{"id":999,"title":"First","name":"Show"}]}"#] + { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target("curated", false, true)); + run.publish().await.unwrap(); + assert!(!run.strm_contents().is_empty()); + server.reply(MOVIES, 200, body); + server.reply(TV, 200, body); + run.publish().await.unwrap(); + let rows = run.xtream_rows().await; + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].title.as_ref(), "Live"); + let m3u = load_m3u_target_storage(&run.context.config, &run.target).await.unwrap(); + assert_eq!(m3u.len(), 1); + let strm = run.strm_contents(); + assert_eq!(strm.len(), 1, "Live survives; only VOD and series files are cleared"); + assert!(strm[0].contains("/101.mkv")); + let watch: std::collections::BTreeSet> = tuliprox_core::utils::binary_deserialize( + &std::fs::read(run.directory.path().join("publication.groups.bin")).unwrap(), + ) + .unwrap(); + assert_eq!(watch.iter().map(AsRef::as_ref).collect::>(), ["Live"]); + let cache = run.context.playlist_state.as_ref().unwrap().data.read().await; + let xtream = cache[&run.target.name].xtream.as_ref().unwrap(); + assert!(xtream.vod.is_empty() && xtream.series.is_empty()); + } +} + +#[tokio::test] +async fn either_https_failure_retains_every_published_file_id_cache_and_watch_in_full_and_curated() { + for policy in ["full", "curated"] { + for (route, status, body) in [ + (MOVIES, 503, "{}"), + (TRAKT, 503, "{}"), + (MOVIES, 200, r#"{"page":1,"total_pages":1,"total_results":1,"results":[{"id":0}]}"#), + ] { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target(policy, true, true)); + let initial = file_snapshot(run.directory.path()); + server.reply(route, status, body); + assert!(run.publish().await.is_err(), "first run must not publish a successful sibling"); + assert_eq!(file_snapshot(run.directory.path()), initial); + assert!(run.cache_signature().await.is_empty()); + server.reply(route, 200, if route == TRAKT { TRAKT_PAGE } else { MOVIE_PAGE }); + run.publish().await.unwrap(); + let files = file_snapshot(run.directory.path()); + let cache = run.cache_signature().await; + assert!(!files.is_empty() && !cache.is_empty()); + server.reply(route, status, body); + assert!(run.publish().await.is_err()); + assert_eq!(file_snapshot(run.directory.path()), files); + assert_eq!(run.cache_signature().await, cache); + } + } +} + +#[tokio::test] +async fn legacy_to_canonical_trakt_migration_retains_published_virtual_ids_and_aliases() { + let server = DiscoveryServer::start().await; + let mut canonical = target("curated", true, true); + canonical.curation.as_mut().unwrap().tmdb = None; + let mut legacy = canonical.clone(); + let curation = legacy.curation.take().unwrap(); + let mut value = serde_json::to_value(curation.trakt.unwrap()).unwrap(); + value["catalog_selection"] = serde_json::to_value(curation.catalog_selection).unwrap(); + for output in &mut legacy.output { + if let TargetOutputDto::Xtream(xtream) = output { + xtream.trakt = Some(serde_json::from_value(value.clone()).unwrap()); + } + } + legacy.prepare(1, None, None).unwrap(); + let mut run = Publication::new(&server, &legacy); + run.publish().await.unwrap(); + let before = run.identity_signature().await; + run.target = ConfigTarget::from(&canonical); + run.publish().await.unwrap(); + assert_eq!(run.identity_signature().await, before); + assert_eq!(server.requests.lock().unwrap().len(), 2); +} diff --git a/backend/processing/src/processor/playlist/tests/curation_publication/client_diagnostics.rs b/backend/processing/src/processor/playlist/tests/curation_publication/client_diagnostics.rs new file mode 100644 index 000000000..86745363f --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/curation_publication/client_diagnostics.rs @@ -0,0 +1,108 @@ +use super::*; +use target::build_target_tmdb_client; +use tuliprox_core::{model::ProxyConfig, utils::network::request::create_tmdb_client}; + +#[tokio::test] +async fn curation_tmdb_client_construction_reports_only_safe_phases_and_keeps_publication_closed() { + for policy in ["full", "curated"] { + for phase in ["phase=profile_configuration", "phase=client_build"] { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target(policy, true, true)); + for previously_published in [false, true] { + if previously_published { + run.publish().await.unwrap(); + } + let files = file_snapshot(run.directory.path()); + let cache = run.cache_signature().await; + assert_eq!(!cache.is_empty(), previously_published); + let before = server.requests.lock().unwrap().len(); + let config = app_config(run.directory.path()); + let mut settings = config.config.load().as_ref().clone(); + if phase == "phase=profile_configuration" { + settings.proxy = Some(ProxyConfig { + url: "http://[synthetic-proxy-secret".into(), + username: Some("synthetic-user-secret".into()), + password: Some("synthetic-password-secret".into()), + }); + } + config.config.store(Arc::new(settings)); + let mut diagnostics = Vec::new(); + let tmdb_client = build_target_tmdb_client( + &run.target, + || { + create_tmdb_client(&config).map(|builder| { + // A controlled reqwest build error, not a new runtime fallback. + builder.user_agent("synthetic-header-secret\ninvalid") + }) + }, + |label| diagnostics.push(label), + ); + assert!(tmdb_client.is_none()); + // Exact allowlist, independent of Display/source and global log sanitization: + // no sanitizer is invoked between construction failure and this reporter. + assert_eq!(diagnostics, [phase]); + assert!(!diagnostics[0].contains("synthetic-")); + let mut changed = catalog(); + changed[0].channels[0].header.title = "Must not publish".intern(); + let prepared = PreparedTarget { + target: run.target.clone(), + playlist: changed, + epg: Vec::new(), + processing: PipelineStats::default(), + accepted_empty_clusters: ClusterFlags::empty(), + library_empty: tuliprox_repository::LibraryEmptyPublication::None, + }; + let (result, errors) = tokio::time::timeout( + Duration::from_secs(5), + target::finalize_prepared_target_with_tmdb( + Arc::clone(&run.context), + prepared, + tmdb_client.as_ref(), + ), + ) + .await + .unwrap(); + assert!(result.is_err()); + assert!(errors.is_empty()); + assert_eq!(file_snapshot(run.directory.path()), files); + assert_eq!(run.cache_signature().await, cache); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before + 1, "complete Trakt sibling cannot authorize publication"); + assert!(requests[before].starts_with(&format!("GET {TRAKT} "))); + } + } + } +} + +#[test] +fn curation_tmdb_disabled_or_selectorless_skips_construction_and_diagnostics() { + for state in ["absent", "block_disabled", "source_disabled", "source_absent", "no_selectors"] { + let mut dto = target("full", false, true); + match state { + "absent" => dto.curation = None, + "block_disabled" => dto.curation.as_mut().unwrap().enabled = false, + "source_disabled" => dto.curation.as_mut().unwrap().tmdb.as_mut().unwrap().enabled = false, + "source_absent" => dto.curation.as_mut().unwrap().tmdb = None, + "no_selectors" => dto.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending.clear(), + _ => unreachable!(), + } + assert!(build_target_tmdb_client( + &ConfigTarget::from(&dto), + || panic!("{state} must not construct a client"), + |_| panic!("{state} must not emit a construction diagnostic"), + ) + .is_none()); + } +} + +#[test] +fn curation_tmdb_successful_construction_has_no_failure_diagnostic() { + let directory = tempdir().unwrap(); + let config = app_config(directory.path()); + assert!(build_target_tmdb_client( + &ConfigTarget::from(&target("full", false, true)), + || create_tmdb_client(&config), + |_| panic!("successful construction is not a failure"), + ) + .is_some()); +} diff --git a/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs new file mode 100644 index 000000000..f49654598 --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/curation_publication/item_limit.rs @@ -0,0 +1,314 @@ +use super::*; + +fn changed_catalog() -> Vec { + let mut playlist = catalog(); + for group in &mut playlist { + for item in &mut group.channels { + item.header.title = format!("Candidate {}", item.header.title).intern(); + item.header.url = format!("http://candidate.invalid/{}.mkv", item.header.id).intern(); + } + } + playlist +} + +fn fail_late(server: &DiscoveryServer, failure: &str) { + match failure { + "status" => server.reply(MOVIES_2, 503, "sensitive-body-must-not-appear"), + "wire" => { + server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":2,"results":[{"id":7},{"id":0}]}"#) + } + "no-progress" => { + server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":1,"results":[{"id":8}]}"#) + } + "empty" => server.reply(MOVIES_2, 200, r#"{"page":2,"total_pages":2,"total_results":0,"results":[]}"#), + "body" => server.reply_raw( + MOVIES_2, + format!("HTTP/1.1 200 OK\r\nContent-Length: 10000\r\nConnection: close\r\n\r\n{MOVIE_PAGE_2}").into_bytes(), + ), + "gzip" | "gzip-transport" | "gzip-suffix" => { + let mut compressed = tuliprox_core::utils::compression_utils::compress_string(MOVIE_PAGE_2).unwrap(); + if failure == "gzip" { + compressed.truncate(compressed.len() - 8); + } else if failure == "gzip-suffix" { + let mut suffix = tuliprox_core::utils::compression_utils::compress_string(" ").unwrap(); + suffix.truncate(suffix.len() - 8); + compressed.extend(suffix); + } + let advertised = compressed.len() + usize::from(failure == "gzip-transport"); + let mut response = format!("HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\nContent-Length: {advertised}\r\nConnection: close\r\n\r\n").into_bytes(); + response.extend(compressed); + server.reply_raw(MOVIES_2, response); + } + "bytes" => { + server.reply(MOVIES_2, 200, &format!("{MOVIE_PAGE_2}{}", " ".repeat(1024 * 1024 + 1 - MOVIE_PAGE_2.len()))) + } + "requests" => { + for p in 1..=33 { + let id = if p == 1 { + 8 + } else if p == 2 { + 7 + } else { + 1000 + p + }; + server.reply( + &format!("/3/trending/movie/week?language=en-US&page={p}"), + 200, + &json!({"page":p,"total_pages":33,"total_results":1,"results":[{"id":id}]}).to_string(), + ); + } + } + _ => panic!("unknown fixture failure"), + } +} + +#[tokio::test] +async fn curation_late_failure_keeps_all_files_epg_cache_mapping_and_watches_in_both_policies() { + for policy in ["full", "curated"] { + for failure in [ + "status", + "wire", + "no-progress", + "empty", + "body", + "gzip", + "gzip-transport", + "gzip-suffix", + "bytes", + "requests", + ] { + let server = DiscoveryServer::start().await; + let mut dto = target(policy, true, true); + // A selection-only selector is just as required as its successful projecting siblings. + dto.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending[0].create_xtream_category = false; + let run = Publication::new(&server, &dto); + let successful_replies = server.replies.lock().unwrap().clone(); + let initial = file_snapshot(run.directory.path()); + fail_late(&server, failure); + assert!(run.publish_catalog(changed_catalog()).await.is_err(), "{policy}/{failure}: first refresh"); + assert_eq!(file_snapshot(run.directory.path()), initial); + assert!(run.context.playlist_state.as_ref().unwrap().data.read().await.is_empty()); + assert!(run.cache_signature().await.is_empty()); + let count = server.requests.lock().unwrap().len(); + assert_eq!( + count, + if failure == "requests" { 34 } else { 4 }, + "Trakt and TV are evaluated too, but not published" + ); + *server.replies.lock().unwrap() = successful_replies; + run.publish().await.unwrap(); + let files = file_snapshot(run.directory.path()); + assert!( + files.keys().any(|path| path.to_string_lossy().contains("epg")), + "EPG must be in the retention inventory: {:?}", + files.keys().collect::>() + ); + let cache = run.cache_signature().await; + assert!(cache.iter().any(|entry| entry.starts_with("mapping:"))); + let watch_path = run.directory.path().join("publication.groups.bin"); + let watch_before: std::collections::BTreeSet> = + tuliprox_core::utils::binary_deserialize(&std::fs::read(&watch_path).unwrap()).unwrap(); + assert!(!watch_before.is_empty()); + fail_late(&server, failure); + assert!( + run.publish_catalog(changed_catalog()).await.is_err(), + "{policy}/{failure}: refresh with prior state" + ); + assert_eq!(file_snapshot(run.directory.path()), files, "inventory and bytes, not just file sizes"); + assert_eq!(run.cache_signature().await, cache, "all cached rows and mapping records, including timestamps"); + let watch_after: std::collections::BTreeSet> = + tuliprox_core::utils::binary_deserialize(&std::fs::read(&watch_path).unwrap()).unwrap(); + assert_eq!(watch_before, watch_after); + } + } +} + +#[tokio::test] +async fn curation_shared_batch_exhaustion_marks_pending_selectors_required_without_129th_get() { + for policy in ["full", "curated"] { + let server = DiscoveryServer::start().await; + let dto = target(policy, false, true); + let mut run = Publication::new(&server, &dto); + let ordinary = run.target.clone(); + let source = run.target.curation.as_mut().unwrap().tmdb.as_mut().unwrap(); + let mut selector = source.trending[0].clone(); + selector.limit = 1; + selector.create_xtream_category = false; + source.trending = vec![selector; 129]; + let failing = run.target.clone(); + let initial = file_snapshot(run.directory.path()); + assert!(run.publish().await.is_err()); + assert_eq!(server.requests.lock().unwrap().len(), 128); + assert_eq!(file_snapshot(run.directory.path()), initial); + assert!(run.cache_signature().await.is_empty()); + run.target = ordinary; + run.publish().await.unwrap(); + let before = file_snapshot(run.directory.path()); + let cache = run.cache_signature().await; + let requests = server.requests.lock().unwrap().len(); + run.target = failing; + assert!(run.publish_catalog(changed_catalog()).await.is_err()); + assert_eq!(server.requests.lock().unwrap().len() - requests, 128); + assert_eq!(file_snapshot(run.directory.path()), before); + assert_eq!(run.cache_signature().await, cache); + } +} + +#[tokio::test] +async fn curation_limit_changes_keep_surviving_ids_and_independent_daily_weekly_memberships() { + const DAILY: &str = "/3/trending/movie/day?language=en-US&page=1"; + for include_base in [false, true] { + let server = DiscoveryServer::start().await; + let mut dto = target("curated", false, true); + let curation = dto.curation.as_mut().unwrap(); + curation.include_xtream_base_categories = include_base; + let source = curation.tmdb.as_mut().unwrap(); + let mut daily = source.trending[0].clone(); + daily.time_window = shared::model::TmdbTrendingTimeWindow::Day; + daily.category_name = Some("TMDB Daily".into()); + daily.limit = 2; + source.trending.push(daily); + let mut run = Publication::new(&server, &dto); + server.reply(DAILY, 200, r#"{"page":1,"total_pages":1,"total_results":2,"results":[{"id":8},{"id":7}]}"#); + let mut prior = BTreeMap::new(); + for (limit, ids) in [(1, vec![7, 7, 8]), (2, vec![7, 7, 8]), (2, vec![8, 7]), (1, vec![8, 7])] { + run.target.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending[0].limit = limit; + server.reply(MOVIES, 200, &json!({"page":1,"total_pages":9,"total_results":ids.len(),"results":ids.iter().map(|id| json!({"id":id})).collect::>()} ).to_string()); + run.publish().await.unwrap(); + let rows = run.xtream_rows().await; + let alive: BTreeMap<_, _> = + rows.iter().map(|row| ((row.group.to_string(), row.title.to_string()), row.virtual_id)).collect(); + for (subject, id) in &prior { + if let Some(current) = alive.get(subject) { + assert_eq!(current, id, "surviving subject/alias {subject:?}"); + } + } + prior = alive; + let identities = run.identity_signature().await; + for title in ["Show", "Show S01E01"] { + let uuid = hash_string(&format!("trakt-category:TMDB TV:{}", hash_string(title))).to_string(); + assert!(identities.contains_key(&uuid)); + } + assert_eq!(rows.iter().any(|row| row.group.as_ref() == "Movies"), include_base); + let mut weekly: Vec<_> = rows.iter().filter(|row| row.group.as_ref() == "TMDB Movies").collect(); + weekly.sort_by_key(|row| row.source_ordinal); + assert_eq!(weekly[0].title.as_ref(), if ids[0] == 7 { "First" } else { "Second" }); + let mut daily: Vec<_> = rows.iter().filter(|row| row.group.as_ref() == "TMDB Daily").collect(); + daily.sort_by_key(|row| row.source_ordinal); + assert_eq!( + daily.iter().map(|row| row.title.as_ref()).collect::>(), + ["Second", "First", "Local alias"] + ); + let m3u = load_m3u_target_storage(&run.context.config, &run.target).await.unwrap(); + for title in ["First", "Local alias", "Second", "Show S01E01"] { + assert_eq!(m3u.iter().filter(|(_, item)| item.title.as_ref() == title).count(), 1); + } + assert!(m3u.iter().all(|(_, item)| !item.group.starts_with("TMDB"))); + } + assert_eq!(server.requests.lock().unwrap().len(), 12, "no extra page after N despite total_pages=9"); + } +} + +#[tokio::test] +async fn curation_full_empty_or_no_match_removes_projections_not_ordinary_catalog() { + for body in [EMPTY, r#"{"page":1,"total_pages":99,"total_results":2,"results":[{"id":999},{"id":7}]}"#] { + let server = DiscoveryServer::start().await; + let mut run = Publication::new(&server, &target("full", false, true)); + run.publish().await.unwrap(); + for selector in &mut run.target.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending { + selector.limit = 1; + } + server.reply(MOVIES, 200, body); + server.reply(TV, 200, body); + run.publish().await.unwrap(); + let rows = run.xtream_rows().await; + assert!(rows.iter().all(|row| !row.group.starts_with("TMDB"))); + assert!(rows.iter().any(|row| row.title.as_ref() == "Unselected")); + assert!(rows.iter().any(|row| row.title.as_ref() == "Live")); + assert!(rows.iter().any(|row| row.title.as_ref() == "Other show")); + assert_eq!( + server.requests.lock().unwrap().len(), + 5, + "do not seek N matches beyond the first unmatched reference" + ); + } +} + +#[tokio::test] +async fn curation_profile_requires_the_fixture_ca_and_has_no_unconfigured_fallback() { + let server = DiscoveryServer::start().await; + let mut run = Publication::new(&server, &target("full", false, true)); + run.tmdb_client = tuliprox_core::utils::network::request::create_tmdb_client(&run.context.config) + .unwrap() + .no_proxy() + .resolve("api.themoviedb.org", server.address) + .build() + .unwrap(); + assert!(run.publish().await.is_err(), "untrusted CA fails rather than disabling TLS verification"); + assert!(server.requests.lock().unwrap().is_empty()); + assert!(file_snapshot(run.directory.path()).is_empty()); +} + +#[tokio::test] +async fn curation_production_wiring_does_not_fall_back_to_the_generic_client() { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target("full", false, true)); + let mut config = (*run.context.config.config.load_full()).clone(); + config.proxy = + Some(tuliprox_core::model::ProxyConfig { url: "invalid proxy".into(), username: None, password: None }); + run.context.config.config.store(Arc::new(config)); + let prepared = PreparedTarget { + target: run.target.clone(), + playlist: catalog(), + epg: Vec::new(), + processing: PipelineStats::default(), + accepted_empty_clusters: ClusterFlags::empty(), + library_empty: tuliprox_repository::LibraryEmptyPublication::None, + }; + let (result, _) = finalize_prepared_target(Arc::clone(&run.context), prepared).await; + assert!(result.is_err()); + assert!( + server.requests.lock().unwrap().is_empty(), + "generic client has the fixture CA and could succeed, but is not used" + ); + assert!(run.cache_signature().await.is_empty()); + assert!(file_snapshot(run.directory.path()).is_empty()); +} + +#[tokio::test] +async fn curation_does_not_authorize_empty_inputs_and_authorized_empty_still_waits_for_all_selectors() { + for authorized in [false, true] { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target("curated", false, true)); + server.reply(MOVIES_2, 503, "unavailable"); + let prepared = PreparedTarget { + target: run.target.clone(), + playlist: Vec::new(), + epg: Vec::new(), + processing: PipelineStats::default(), + accepted_empty_clusters: if authorized { ClusterFlags::Vod } else { ClusterFlags::empty() }, + library_empty: tuliprox_repository::LibraryEmptyPublication::None, + }; + let (result, _) = + target::finalize_prepared_target_with_tmdb(Arc::clone(&run.context), prepared, Some(&run.tmdb_client)) + .await; + assert_eq!(result.is_err(), authorized); + assert_eq!(server.requests.lock().unwrap().len(), if authorized { 3 } else { 0 }); + assert!(file_snapshot(run.directory.path()).is_empty()); + assert!(run.cache_signature().await.is_empty()); + } +} + +#[tokio::test] +async fn curation_post_admission_writer_failure_remains_best_effort_not_a_rollback() { + let server = DiscoveryServer::start().await; + let run = Publication::new(&server, &target("curated", false, true)); + std::fs::create_dir(run.directory.path().join("published.m3u")).unwrap(); + assert!(run.publish().await.is_err()); + assert!(!run.xtream_rows().await.is_empty(), "other admitted output was still written"); + assert_eq!(server.requests.lock().unwrap().len(), 3); + assert!( + !run.directory.path().join("publication.groups.bin").exists(), + "watches do not finalize a failed writer run" + ); +} diff --git a/backend/processing/src/processor/playlist/tests/curation_publication/tls_profile.rs b/backend/processing/src/processor/playlist/tests/curation_publication/tls_profile.rs new file mode 100644 index 000000000..821342f0b --- /dev/null +++ b/backend/processing/src/processor/playlist/tests/curation_publication/tls_profile.rs @@ -0,0 +1,106 @@ +//! Exercise the configured profile against real TLS, without changing host trust or contacting TMDB. +use super::*; +use reqwest::header::AUTHORIZATION; +use tuliprox_core::utils::network::request::{create_client, create_tmdb_client}; + +fn insecure_config(directory: &std::path::Path) -> Arc { + let config = app_config(directory); + let mut settings = config.config.load().as_ref().clone(); + settings.accept_insecure_ssl_certificates = true; + config.config.store(Arc::new(settings)); + config +} + +fn local_client(builder: reqwest::ClientBuilder, server: &DiscoveryServer, hostname: &str) -> reqwest::Client { + builder.no_proxy().resolve(hostname, server.address).timeout(Duration::from_secs(3)).build().unwrap() +} + +#[tokio::test] +async fn tmdb_profile_tls_rejects_untrusted_certificate_even_with_global_insecure_flag() { + let server = DiscoveryServer::start().await; + let directory = tempdir().unwrap(); + let config = insecure_config(directory.path()); + let client = local_client(create_tmdb_client(&config).unwrap(), &server, "api.themoviedb.org"); + let result = client + .get(format!("https://api.themoviedb.org{MOVIES}")) + .header(AUTHORIZATION, "Bearer synthetic-tls-secret") + .send() + .await; + assert!(result.is_err(), "discovery must not inherit the global TLS bypass"); + assert!(result.unwrap_err().is_connect()); + assert!(server.requests.lock().unwrap().is_empty(), "TCP/TLS is not an HTTP request or Bearer disclosure"); +} + +#[tokio::test] +async fn tmdb_profile_tls_preserves_trusted_ca_and_checks_hostname_with_global_insecure_flag() { + let server = DiscoveryServer::start().await; + let directory = tempdir().unwrap(); + let config = insecure_config(directory.path()); + for (hostname, trusted_name) in [("api.themoviedb.org", true), ("wrong-name.invalid", false)] { + let client = local_client( + create_tmdb_client(&config).unwrap().add_root_certificate(server.certificate.clone()), + &server, + hostname, + ); + let before = server.requests.lock().unwrap().len(); + let result = client + .get(format!("https://{hostname}{MOVIES}")) + .header(AUTHORIZATION, "Bearer synthetic-tls-secret") + .send() + .await; + if trusted_name { + assert_eq!(result.unwrap().text().await.unwrap(), MOVIE_PAGE); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before + 1); + assert!(requests[before].to_lowercase().contains("authorization: bearer synthetic-tls-secret")); + } else { + assert!(result.unwrap_err().is_connect()); + assert_eq!(server.requests.lock().unwrap().len(), before, "wrong hostname must not receive HTTP"); + } + } +} + +#[tokio::test] +async fn tmdb_profile_tls_does_not_change_generic_clients_global_insecure_policy() { + let server = DiscoveryServer::start().await; + let directory = tempdir().unwrap(); + let config = insecure_config(directory.path()); + let client = local_client(create_client(&config), &server, "api.trakt.tv"); + let response = client.get(format!("https://api.trakt.tv{TRAKT}")).send().await.unwrap(); + assert_eq!(response.text().await.unwrap(), TRAKT_PAGE); + assert_eq!(server.requests.lock().unwrap().len(), 1, "generic profile still accepts the untrusted certificate"); +} + +#[tokio::test] +async fn tmdb_profile_tls_failure_blocks_publication_and_retains_artifacts_even_with_complete_trakt() { + for policy in ["full", "curated"] { + let server = DiscoveryServer::start().await; + let mut run = Publication::new(&server, &target(policy, true, true)); + let config = insecure_config(run.directory.path()); + let rejected = local_client(create_tmdb_client(&config).unwrap(), &server, "api.themoviedb.org"); + let trusted = local_client( + create_tmdb_client(&config).unwrap().add_root_certificate(server.certificate.clone()), + &server, + "api.themoviedb.org", + ); + for previously_published in [false, true] { + if previously_published { + run.tmdb_client = trusted.clone(); + run.publish().await.unwrap(); + } + let files = file_snapshot(run.directory.path()); + let cache = run.cache_signature().await; + assert_eq!(!cache.is_empty(), previously_published); + let before = server.requests.lock().unwrap().len(); + run.tmdb_client = rejected.clone(); + let mut changed = catalog(); + changed[0].channels[0].header.title = "Must not publish".intern(); + assert!(run.publish_catalog(changed).await.is_err()); + assert_eq!(file_snapshot(run.directory.path()), files, "all files, IDs and watches retained"); + assert_eq!(run.cache_signature().await, cache, "no partial cache publication"); + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), before + 1, "only the required Trakt sibling reached HTTP; no fallback"); + assert!(requests[before].starts_with(&format!("GET {TRAKT} "))); + } + } +} diff --git a/backend/repository/src/epg_repository.rs b/backend/repository/src/epg_repository.rs index 5cc3971e2..319df8f53 100644 --- a/backend/repository/src/epg_repository.rs +++ b/backend/repository/src/epg_repository.rs @@ -252,6 +252,7 @@ mod tests { fn target_with_m3u_and_xtream() -> ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: "ics-target".to_string(), diff --git a/backend/repository/src/playlist_repository.rs b/backend/repository/src/playlist_repository.rs index 0098c433d..521e88427 100644 --- a/backend/repository/src/playlist_repository.rs +++ b/backend/repository/src/playlist_repository.rs @@ -1409,6 +1409,7 @@ mod tests { fn target_with_options(name: &str, output: Vec, use_memory_cache: bool) -> ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: name.to_string(), diff --git a/backend/repository/src/xtream_repository.rs b/backend/repository/src/xtream_repository.rs index 491b22d1e..4b5a9d765 100644 --- a/backend/repository/src/xtream_repository.rs +++ b/backend/repository/src/xtream_repository.rs @@ -2955,6 +2955,7 @@ mod tests { fn target_writer_config() -> ConfigTarget { ConfigTarget { + curation: None, id: 1, enabled: true, name: "target-empty-cluster-guard".to_string(), diff --git a/config/source.yml b/config/source.yml index c5c77f0a4..044d721cf 100644 --- a/config/source.yml +++ b/config/source.yml @@ -35,6 +35,31 @@ sources: mpeg_ts: true mapping: - iptv-org + # Link this example to your authorized VOD/Series input with TMDB IDs, not just the demo Live feed. + - name: tmdb-discovery-example + enabled: false + curation: + # Enable target and curation after providing your own application Read Access Token. + enabled: false + catalog_selection: full + include_xtream_base_categories: true + tmdb: + api: + # See docs for the TMDB_READ_ACCESS_TOKEN environment-variable example; no metadata-key fallback. + access_token: "" + trending: + - kind: movie + time_window: week + limit: 100 # Unique remote references, not local matches; optional (1..=500). + category_name: TMDB Weekly Movies + - kind: tv + time_window: day + limit: 100 + category_name: TMDB Daily TV + output: + - type: xtream + - type: m3u + filename: tmdb-discovery.m3u - name: iptv-trakt-example output: - type: xtream diff --git a/docs/src/configuration/config.md b/docs/src/configuration/config.md index 4e0c3175d..23d92af23 100644 --- a/docs/src/configuration/config.md +++ b/docs/src/configuration/config.md @@ -68,6 +68,10 @@ metadata_update: | `custom_stream_response_enabled` | Bool | No | `true` | Enables configured custom MPEG-TS fallback videos for stream errors. When `false`, factories skip the video body and call sites return `custom_stream_response_error_status`; useful behind reverse proxies that intercept 4xx/5xx responses instead of keeping fallback sockets open. | | `custom_stream_response_error_status` | Int | No | `502` | HTTP status code returned when `custom_stream_response_enabled` is `false`. Must be a 4xx or 5xx code (the `prepare()` step rejects anything outside that range; `0` is silently clamped to `502`). Common choices: `404` (channel not found), `502` (bad gateway — upstream failed), `503` (service unavailable — overloaded). | +**TMDB discovery exception:** [TMDB curation discovery](source.md#target-wide-curation-curation) always verifies TLS +certificates and hostnames, even with `accept_insecure_ssl_certificates: true`. TLS-inspecting proxies require valid CA +trust to complete discovery. Trakt, metadata and playback clients retain this flag's behavior. + --- #### ⚠️ Important: `config_hot_reload` & Bind-Mounts diff --git a/docs/src/configuration/mapping-dsl.md b/docs/src/configuration/mapping-dsl.md index 4c914a53c..862448654 100644 --- a/docs/src/configuration/mapping-dsl.md +++ b/docs/src/configuration/mapping-dsl.md @@ -108,7 +108,7 @@ Collect all processed inputs for the target Configured favourite aliases -> Merge groups -> Post-merge content deduplication | v -Evaluate required Trakt selectors -> Select catalog -> Derive base and Xtream appearances +Evaluate required curation selectors (Trakt and/or TMDB) -> Select catalog -> Derive base and Xtream appearances | v Sort -> Assign channel numbers -> Mapping counters -> Persist/output filters -> Persistence -> Watches @@ -117,7 +117,7 @@ Sort -> Assign channel numbers -> Mapping counters -> Persist/output filters -> `processing_order` controls only the order of filter (`F`), rename (`R`), and the default mapping position (`M`). `after_epg` is a separate stage outside that configurable pipe. Mapping counters are also outside the pipe and run on each finalized appearance after the target has merged and deduplicated the curation-eligible catalog, then sorted and -assigned initial channel numbers. Trakt curation never changes which mapper stage runs. +assigned initial channel numbers. Target curation never changes which mapper stage runs. The configured order and mapping-stage indexes are compiled when configuration is loaded. During a refresh, the F/R/M pipe materializes each input into one item buffer, preserves the historical grouping order between stages, and rebuilds diff --git a/docs/src/configuration/source.md b/docs/src/configuration/source.md index 3dcd6aaa3..ef4d0bc1c 100644 --- a/docs/src/configuration/source.md +++ b/docs/src/configuration/source.md @@ -1369,6 +1369,7 @@ sources: | `sort` | Object | No | | Defines ordering for groups and channels after transformations. This affects the final playlist structure seen by clients and can significantly improve navigation quality in IPTV players. | | `options` | Object | No | | Target-level behavior switches such as logo suppression, duplicate removal, and shared live-stream handling. These options influence memory usage, playlist cleanliness, and reverse-proxy behavior. | | `output` | List | Yes | | Mandatory list of output formats. A single target can generate multiple output representations (e.g., `xtream`, `m3u`, `strm`, `hdhomerun`) from the same transformed result set. | +| `curation` | Object | No | | Optional target-wide catalog policy with Trakt and/or TMDB Trending sources. Mutually exclusive with legacy `output[].trakt`; see [target-wide curation](#target-wide-curation-curation). | | `favourites` | List | No | | Duplicates final transformed channels into dedicated favorite groups after processing is complete. This adds curated views without changing the original group structure. | | `watch` | List | No | | Defines watched group patterns. If matching groups change during updates, Tuliprox emits Messaging events so operational changes become observable automatically. | | `use_memory_cache` | Bool | No | `false` | If enabled, the final compiled playlist is cached in RAM. This reduces disk access and improves delivery speed, especially for M3U downloads, but increases memory consumption. | @@ -1419,7 +1420,7 @@ filter: ``` `processing` runs at the normal `F` position. The target then applies favourites, merges groups, performs post-merge -content deduplication, evaluates any configured Trakt selectors, and derives the base and Xtream appearance views. +content deduplication, evaluates any configured curation selectors (Trakt and/or TMDB), and derives the base and Xtream appearance views. `persist` runs after each view is sorted, numbered, and counted, immediately before output filters, watch evaluation, and persistence. Output-level filters remain plain strings and have no configurable stage. @@ -1794,6 +1795,205 @@ visible IDs change. --- +### Target-wide curation (`curation`) + +Curation is operator-configured discovery over content already available in this target. It neither imports new playback +sources nor uses personal watch history, progress, OAuth or recommendations. TMDB Trending works without a Trakt Client ID. +Legacy [`output[].trakt`](#trakt-object-in-xtream-output) remains supported; existing configurations need no migration. + +Start with `catalog_selection: full` to retain the ordinary catalog and add Xtream categories for locally available matches: + +```yaml +# A target under sources[].targets, linked to your authorized movie/TV inputs. +name: discovery +curation: + catalog_selection: full + include_xtream_base_categories: true + tmdb: + api: + access_token: "${env:TMDB_READ_ACCESS_TOKEN}" + trending: + - kind: movie + time_window: week + limit: 100 + category_name: TMDB Weekly Movies + - kind: tv + time_window: day + limit: 100 + category_name: TMDB Daily TV +output: + - type: xtream + - type: m3u + filename: discovery.m3u +``` + +Obtain your application's **API Read Access Token** from [TMDB API settings](https://www.themoviedb.org/settings/api). +Configure the raw token without a `Bearer` prefix, preferably through an environment variable. It is sent as a sensitive +Authorization header, never as a query parameter. There is no bundled discovery token, `api_key` alias, configurable TMDB +origin, or fallback to `metadata_update.tmdb`. Discovery credentials and enablement are independent of metadata enrichment. +Missing/blank credentials or values that cannot form an HTTP header leave configuration readable but make an enabled +selector unavailable without a request. Rejected/expired credentials also fail the refresh, rather than yielding an empty feed. + +#### Curation policy and source parameters + +| Parameter | Default | Meaning | +| --- | --- | --- | +| `enabled` | `true` | Enables the whole curation block. | +| `catalog_selection` | `full` | `full` retains the eligible catalog; `curated` keeps the union of selected VOD/series subjects and series children, plus Live. | +| `include_xtream_base_categories` | `true` | Includes ordinary VOD/Series groups in Xtream alongside projections. Does not affect Live or M3U/STRM grouping. | +| `trakt` | absent | Optional source with `enabled`, `api`, `lists`, `charts`; reuses the [Trakt selector parameters](#trakt-parameters). | +| `tmdb` | absent | Optional TMDB Trending source. | +| `trakt.enabled`, `tmdb.enabled` | `true` | Disabled sources make no requests and contribute no required selectors. | +| `tmdb.api.access_token` | empty | Explicit TMDB application Read Access Token, required for an active fetch. | +| `tmdb.trending` | `[]` | Configured Trending selectors. Each active entry is required. | +| `tmdb.trending[].kind` | required | `movie` or `tv`; no mixed/person endpoint. | +| `tmdb.trending[].time_window` | required | `day` or `week`. | +| `tmdb.trending[].limit` | `100` | Integer **1..=500**: up to this many unique remote references, before local matching. | +| `tmdb.trending[].create_xtream_category` | `true` | Projects a category in Xtream; false makes the selector selection-only. | +| `tmdb.trending[].category_name` | absent | Nonblank name required for an active projection; retained but unused for selection-only entries. | + +There is one catalog policy per target, not one per provider or output. `curation.trakt` does not accept its own +`catalog_selection` or `include_xtream_base_categories`. An active projection, or suppression of Xtream base categories, +requires an Xtream output. Projection names involving TMDB must be unique within each media cluster after trimming, +lowercasing and accent removal, including overlaps with Trakt selectors. A Trakt `both` list reserves both clusters; +selection-only and disabled sources reserve neither. Existing Trakt-only and ordinary base-group merging is unchanged. +Category names still participate in alias identity: renaming them does not promise stable virtual IDs. + +An enabled non-default policy without active selectors is rejected. An all-default source-less block is a no-op. +Disabling the whole block preserves incomplete category/credential edits without executing them. Required enum fields +(`kind`, `time_window`), unknown-field checks and declaration ownership checks still apply to disabled drafts. `limit` must +remain an integer in range even when the target, block or source is disabled: null, zero, negatives, fractions, quoted +numbers and overflow are errors, not defaults or clamped values. Saving omits the default 100 and retains non-default limits. + +#### Bounded discovery, matching and failure behavior + +TMDB requests `/3/trending/movie/{day|week}` or `/3/trending/tv/{day|week}`, with `language=en-US&page=p`, sequentially +from page 1. Pagination is internal transport, not operator configuration. There is no remote `limit` parameter, Discover +filtering or details lookup. N counts unique **(movie/TV kind, positive TMDB ID)** references, not local titles, versions, +aliases or episodes. An unavailable local title still consumes a reference; Tuliprox does not keep searching for N matches. + +Selection stops at the Nth unique reference, but the entire last response is read and validated, including duplicates and +the suffix beyond N. An invalid record anywhere fails the selector. Duplicates keep the first text and rank. Rank is the +observed row ordinal across actual page lengths, including duplicates, not an ordinal among unique IDs or a fixed page-size +calculation. No extra page is requested after N or a valid end. A valid end before N is also complete. + +Each response must echo the requested page and have coherent totals. A short page alone is not an end. Totals may change; +the current page's last-page indication is used without assuming a remote snapshot. An empty response is valid only on +page 1 with zero results and total pages 0 or 1. A later empty page, or a later page with no new IDs (even the last page), +fails rather than publishing a partial prefix. This deliberately conservative rule can reject a legitimately overlapping +page. Trending pagination has empirical support; TMDB's public Trending references do not currently document the `page` +query parameter. If its behavior changes, these checks fail closed, not back to first-page discovery. + +Internal guards are independent of `limit` and cannot be configured: + +| Resource | Per request | Per TMDB selector | Shared TMDB acquisition batch per target/run | +| --- | --- | --- | --- | +| Admitted GET attempts | 1 | 32 | 128 | +| Absolute deadline | 15 seconds | 60 seconds | 180 seconds | +| Decoded bytes consumed | 1 MiB | 8 MiB | 32 MiB | + +The dedicated configured HTTP profile preserves proxy/authentication, trusted CAs and connect timeout, but always verifies +TLS certificates and hostnames, even when `accept_insecure_ssl_certificates: true`. A TLS-inspecting proxy with an untrusted +certificate cannot complete TMDB discovery until valid CA trust is established. This exception does not change Trakt, +metadata or playback clients. The profile never follows redirects or replays requests. Client construction failures have +no unconfigured fallback. Attempts are counted before +sending, and consumed decoded bytes include duplicate/suffix rows and failed bodies; size detection may consume one extra +byte on failure, never on success. These are not TLS/compressed-byte or total-memory limits. Deadlines cover headers, +decoding and normalization, do not restart per page/chunk, and are checked before success. They bound TMDB acquisition, +not inputs, Trakt, local matching or the entire refresh. Acquisition finishes before local matching. There is no feed +cache or stale-feed fallback. A valid N may not fit these guards: needing another request or byte after exhaustion fails; +completing exactly at a request/byte boundary succeeds. Do not assume 100 references always require five pages or yield +100 matches. + +Matching is exact by positive TMDB ID **and** movie/TV kind; there is no title/year fuzzy fallback for TMDB. Local items +without a matching ID are not selected. Distinct local subjects sharing an ID remain distinct; duplicate feed entries keep +the first rank. Day/week selectors keep independent ranks and memberships; their base selection is unioned by local +subject, not remote ID. Changing only the limit preserves surviving subject/alias identities when local UUID and category +name remain unchanged. Remote order applies within each projected category, without globally reordering the base catalog. +Existing target sorting and filters still apply afterwards. M3U and STRM consume selected ordinary entries, not the new +Xtream aliases; HDHomeRun follows its existing underlying output. + +All enabled Trakt and TMDB selectors, including selection-only selectors, must complete before publication in **both** +`full` and `curated`. A timeout, resource exhaustion, unsuccessful status, malformed/inconsistent page, invalid ID/media +kind, lack of pagination progress or any redirect prevents changes to finalized target IDs, files, caches and watches. Successful sibling +feeds do not authorize partial publication. The first failed run publishes nothing; later failures retain prior artifacts. +`full` does not make configured discovery optional. Writers remain best-effort after successful curation admission; this +is not a cross-output storage transaction. + +A valid initial empty page and a complete selection with **no local matches are successes**, not errors. In `curated`, either can clear +published VOD/Series while retaining eligible Live. This can happen when the library lacks trending titles or TMDB IDs. +Enrichment is separate: rebuild the target after IDs become available. Start with `full` if this narrowing is not intended. + +#### Selection-only and mixed sources + +M3U/STRM-only targets are supported by explicitly disabling Xtream projection: + +```yaml +name: weekly-m3u +curation: + catalog_selection: curated + tmdb: + api: + access_token: "${env:TMDB_READ_ACCESS_TOKEN}" + trending: + - kind: movie + time_window: week + limit: 100 + create_xtream_category: false +output: + - type: m3u + filename: weekly.m3u +``` + +To combine sources, add a Trakt source inside the same `curation` block. For example: + +```yaml +trakt: + api: + api_key: "${env:TRAKT_CLIENT_ID}" + charts: + - kind: movies + chart: popular + category_name: Trakt Popular Movies + tmdb_only: true +``` + +Both providers read the same eligible pre-projection catalog. In `curated`, their selected subjects are unioned once for +normal outputs; each projected Xtream category retains its own memberships/rank. Either provider failing blocks the run. + +#### Explicit migration and Source Editor + +At most one non-null declaration surface is allowed: `target.curation` **or** legacy `output[].trakt`. Declaring both is +an error even if one is disabled, empty or uses only defaults. Null is equivalent to absence. There is no implicit merge, +precedence or automatic migration. + +To migrate an existing Trakt target deliberately: + +1. Move its whole-block `enabled`, `catalog_selection` and `include_xtream_base_categories` to `target.curation`. +2. Move its `api`, `lists` and `charts` into `curation.trakt`; leave this source enabled (the default). +3. Remove `output[].trakt`. Leave the rest of the output configuration in place. + +This preserves the original policy, including a globally disabled block, and the legacy matching/category identity rules. +Equivalent canonical Trakt-only configurations keep their existing aliases/virtual IDs. Add TMDB only after deliberately +choosing the shared policy; all newly enabled selectors become required. + +The new block is YAML-owned. Source Editor shows a read-only target summary, preserves the block when editing other +fields, and hides legacy Trakt controls on its connected outputs. Edit provider/policy settings in YAML; there is no TMDB +query-builder or migration wizard. Read/save/reload preserves configured meaning and ownership, not YAML formatting or +spelling of defaults/aliases. Backend validation still rejects conflicting declarations introduced by graph edits. + +#### TMDB attribution and usage terms + +The dashboard version card's **Credits** button displays an approved TMDB logo and the required notice: + +> This product uses the TMDB API but is not endorsed or certified by TMDB. + +The developer API is free for non-commercial use with attribution. Commercial deployments must assess TMDB's commercial +licensing requirements; configuring a token does not grant unrestricted commercial use or rights to playback content. +See the [TMDB FAQ](https://developer.themoviedb.org/docs/faq), +[branding guidance](https://www.themoviedb.org/about/logos-attribution) and +[commercial API information](https://www.themoviedb.org/api-for-business). + ### 3.2.7 Output Formats (`output`) A target can be exported to multiple formats simultaneously. The target-level filter, rename, mapping, and sort @@ -1865,6 +2065,9 @@ output: #### `trakt` Object in Xtream Output +This is the compatible legacy declaration. For TMDB or combined sources use +[target-wide `curation`](#target-wide-curation-curation) instead; do not declare both surfaces on one target. + Trakt.tv is an online platform for tracking, organizing, and discovering movies and TV shows. Tuliprox evaluates every configured list and chart against the target's merged, deduplicated VOD and series catalog. The resulting exact-entry memberships can select the target-wide catalog and independently create compatibility alias categories in Xtream. diff --git a/frontend/public/assets/i18n/ar.json b/frontend/public/assets/i18n/ar.json index ad5bcc812..aa68d028e 100644 --- a/frontend/public/assets/i18n/ar.json +++ b/frontend/public/assets/i18n/ar.json @@ -1434,6 +1434,7 @@ "REFRESH": "تحديث", "REGEXP": "تعبير منتظم", "RELEASES": "الإصدارات", + "CREDITS": "الاعتمادات", "REMOVE_DUPLICATES": "إزالة التكرارات", "CLEAR_INVALID_EPG_IDS": "مسح معرفات EPG غير الصالحة", "RENAME": "إعادة تسمية", @@ -1611,6 +1612,8 @@ "TMDB": "TMDB", "TOKEN": "رمز", "TOKEN_TTL_MINS": "TTL الرمز (دقائق)", + "CURATION": "تنسيق المحتوى", + "CURATION_YAML_NOTICE": "تم إعداد تنسيق محتوى الهدف في YAML. تُحفظ هذه الإعدادات عند تعديل الحقول الأخرى؛ عناصر تحكم Trakt القديمة غير متاحة لهذا الهدف.", "TRAKT": "Trakt", "TRAKT_CATALOG_SELECTION": "كتالوج VOD والمسلسلات", "TRAKT_CATALOG_SELECTION_FULL": "الكتالوج الكامل", diff --git a/frontend/public/assets/i18n/en.json b/frontend/public/assets/i18n/en.json index 99450b595..1cd4a15be 100644 --- a/frontend/public/assets/i18n/en.json +++ b/frontend/public/assets/i18n/en.json @@ -1519,6 +1519,7 @@ "REFRESH": "Refresh", "REGEXP": "Regexp", "RELEASES": "Releases", + "CREDITS": "Credits", "REMOVE_DUPLICATES": "Remove duplicates", "CLEAR_INVALID_EPG_IDS": "Clear invalid EPG IDs", "RENAME": "Rename", @@ -1695,6 +1696,8 @@ "TMDB": "TMDB", "TOKEN": "Token", "TOKEN_TTL_MINS": "Token ttl (mins)", + "CURATION": "Curation", + "CURATION_YAML_NOTICE": "Target curation is configured in YAML. These settings are preserved when editing other fields; legacy Trakt controls are unavailable for this target.", "TRAKT": "Trakt", "TRAKT_CATALOG_SELECTION": "VOD and series catalog", "TRAKT_CATALOG_SELECTION_FULL": "Full catalog", diff --git a/frontend/public/assets/i18n/ru.json b/frontend/public/assets/i18n/ru.json index 3d3ddf517..28775700a 100644 --- a/frontend/public/assets/i18n/ru.json +++ b/frontend/public/assets/i18n/ru.json @@ -1375,6 +1375,7 @@ "REFRESH": "Обновить", "REGEXP": "Регулярное выражение", "RELEASES": "Релизы", + "CREDITS": "Благодарности", "REMOVE_DUPLICATES": "Удалить дубликаты", "CLEAR_INVALID_EPG_IDS": "Очистить недействительные EPG ID", "RENAME": "Переименовать", @@ -1519,6 +1520,8 @@ "TMDB": "TMDB", "TOKEN": "Токен", "TOKEN_TTL_MINS": "TTL токена (мин)", + "CURATION": "Подбор контента", + "CURATION_YAML_NOTICE": "Подбор контента для цели настроен в YAML. Эти настройки сохраняются при изменении других полей; прежние настройки Trakt для этой цели недоступны.", "TRAKT": "Trakt", "TRAKT_CATALOG_SELECTION": "Каталог VOD и сериалов", "TRAKT_CATALOG_SELECTION_FULL": "Полный каталог", diff --git a/frontend/public/assets/tmdb-logo.NOTICE b/frontend/public/assets/tmdb-logo.NOTICE new file mode 100644 index 000000000..3e6193bcd --- /dev/null +++ b/frontend/public/assets/tmdb-logo.NOTICE @@ -0,0 +1,13 @@ +# TMDB logo attribution + +`tmdb-logo.svg` is the unmodified approved TMDB short blue logo, retrieved on 2026-09-20. + +- [Official logos and attribution guidance](https://www.themoviedb.org/about/logos-attribution) +- [Original SVG](https://www.themoviedb.org/assets/v4/logos/v2/blue_short-8e7b30f73a4020692ccca9c88bafe5dcb6f8a62a4c6bc55cd9ba82bb2cd95f6c.svg) +- SHA-256: `8e7b30f73a4020692ccca9c88bafe5dcb6f8a62a4c6bc55cd9ba82bb2cd95f6c` +- [API attribution and commercial-use requirements](https://developer.themoviedb.org/docs/faq) + +The TMDB name and logo identify The Movie Database; their use is subject to TMDB's branding terms. +This asset is not a Tuliprox-created logo. Do not recolor, distort, rotate or imply endorsement. + +This product uses the TMDB API but is not endorsed or certified by TMDB. diff --git a/frontend/public/assets/tmdb-logo.svg b/frontend/public/assets/tmdb-logo.svg new file mode 100644 index 000000000..4b21ded32 --- /dev/null +++ b/frontend/public/assets/tmdb-logo.svg @@ -0,0 +1 @@ +Asset 3 \ No newline at end of file diff --git a/frontend/scss/app/_component.scss b/frontend/scss/app/_component.scss index 3a68e2f92..d9cda8854 100644 --- a/frontend/scss/app/_component.scss +++ b/frontend/scss/app/_component.scss @@ -15,6 +15,7 @@ @forward "components/sidebar"; @forward "components/preferences"; @forward "components/dashboard/action_card"; +@forward "components/dashboard/version_action_card"; @forward "components/dashboard/status_card"; @forward "components/dashboard/sparkline"; @forward "components/dashboard/dashboard_view"; diff --git a/frontend/scss/app/components/dashboard/_version_action_card.scss b/frontend/scss/app/components/dashboard/_version_action_card.scss new file mode 100644 index 000000000..95ebdbd14 --- /dev/null +++ b/frontend/scss/app/components/dashboard/_version_action_card.scss @@ -0,0 +1,18 @@ +.tp__custom-dialog.tp__content-dialog:has(.tp__version-credits) { + min-width: 0; + max-width: min(38rem, 90vw); + box-sizing: border-box; +} + +.tp__version-credits { + max-width: 36rem; + + img { + display: block; + max-width: 100%; + height: auto; + } + + &__app-logo { width: 160px; } + &__tmdb-logo { width: 120px; } +} diff --git a/frontend/src/app/components/dashboard/version_action_card.rs b/frontend/src/app/components/dashboard/version_action_card.rs index c565aed76..84a92f153 100644 --- a/frontend/src/app/components/dashboard/version_action_card.rs +++ b/frontend/src/app/components/dashboard/version_action_card.rs @@ -2,10 +2,30 @@ use crate::{ app::components::{ActionCard, TextButton}, hooks::use_service_context, i18n::use_translation, + services::DialogService, }; use gloo_utils::window; use shared::utils::concat_path_leading_slash; -use yew::prelude::*; +use yew::{platform::spawn_local, prelude::*}; + +const TMDB_API_NOTICE: &str = "This product uses the TMDB API but is not endorsed or certified by TMDB."; + +fn asset_url(web_path: Option<&str>, asset: &str) -> String { + web_path.map_or_else(|| asset.to_string(), |path| concat_path_leading_slash(path, asset)) +} + +fn credits_content(title: String, app_logo: String, tmdb_logo: String) -> Html { + html! { +
+ +

{title}

+ + + +

{TMDB_API_NOTICE}

+
+ } +} #[derive(Properties, Clone, PartialEq, Debug)] pub struct VersionActionProps { @@ -17,6 +37,7 @@ pub struct VersionActionProps { pub fn VersionActionCard(props: &VersionActionProps) -> Html { let translate = use_translation(); let services = use_service_context(); + let dialog = use_context::().expect("Dialog service not found"); let handle_url = { let services = services.clone(); @@ -26,18 +47,54 @@ pub fn VersionActionCard(props: &VersionActionProps) -> Html { }) }; - let logo_url = { - let mut url = "/assets/tuliprox-logo.svg".to_owned(); - if let Some(web_path) = services.config.ui_config.web_path.as_ref() { - url = concat_path_leading_slash(web_path, &url); - } - url + let web_path = services.config.ui_config.web_path.as_deref(); + let logo_url = asset_url(web_path, "/assets/tuliprox-logo.svg"); + let handle_credits = { + let app_logo = logo_url.clone(); + let tmdb_logo = asset_url(web_path, "/assets/tmdb-logo.svg"); + let title = translate.t("LABEL.CREDITS"); + Callback::from(move |_| { + let content = credits_content(title.clone(), app_logo.clone(), tmdb_logo.clone()); + let dialog = dialog.clone(); + spawn_local(async move { + let _ = dialog.content(content, None, true).await; + }); + }) }; html! { + } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn credits_assets_respect_the_configured_web_path() { + assert_eq!(asset_url(None, "/assets/tmdb-logo.svg"), "/assets/tmdb-logo.svg"); + assert_eq!(asset_url(Some("/gateway"), "/assets/tmdb-logo.svg"), "/gateway/assets/tmdb-logo.svg"); + assert_eq!(asset_url(Some("/gateway"), "/assets/tuliprox-logo.svg"), "/gateway/assets/tuliprox-logo.svg"); + } + + #[test] + fn credits_content_includes_official_logo_link_and_exact_notice() { + let content = + credits_content("Credits".into(), "/assets/tuliprox-logo.svg".into(), "/assets/tmdb-logo.svg".into()); + let tree = format!("{content:?}"); + for required in [ + TMDB_API_NOTICE, + "/assets/tmdb-logo.svg", + "/assets/tuliprox-logo.svg", + "https://www.themoviedb.org", + "tp-credits-title", + ] { + assert!(tree.contains(required), "missing credit content: {required}"); + } + } +} diff --git a/frontend/src/app/components/source_editor/editor_model.rs b/frontend/src/app/components/source_editor/editor_model.rs index d13cbd30d..0abba528e 100644 --- a/frontend/src/app/components/source_editor/editor_model.rs +++ b/frontend/src/app/components/source_editor/editor_model.rs @@ -235,6 +235,7 @@ pub struct SourceEditorContext { pub on_form_change: Callback<(BlockId, BlockInstance)>, pub open_target_bouquet: Callback, pub bouquet_revision: u64, + pub output_curation_managed: bool, pub edit_mode: UseStateHandle, pub allow_write: bool, } diff --git a/frontend/src/app/components/source_editor/editor_view.rs b/frontend/src/app/components/source_editor/editor_view.rs index 6c3e51a2b..a10f61148 100644 --- a/frontend/src/app/components/source_editor/editor_view.rs +++ b/frontend/src/app/components/source_editor/editor_view.rs @@ -473,6 +473,14 @@ fn input_dedupe_key(input_config: &ConfigInputDto) -> Option { } } +fn output_has_target_curation(state: &EditorState, output_id: BlockId) -> bool { + state.connections.iter().filter(|connection| connection.to == output_id).any(|connection| { + state + .get_block(connection.from) + .is_some_and(|block| matches!(&block.instance, BlockInstance::Target(target) if target.curation.is_some())) + }) +} + fn editor_state_to_sources_config(base_sources: &SourcesConfigDto, editor_state: &EditorState) -> SourcesConfigDto { let mut sources_config = base_sources.clone(); let mut gen_sources: Vec = Vec::new(); @@ -2129,6 +2137,10 @@ pub fn SourceEditor(props: &SourceEditorProps) -> Html { on_form_change: form_changed, open_target_bouquet, bouquet_revision: *bouquet_revision, + output_curation_managed: match &*edit_mode { + EditMode::Active(block) => output_has_target_curation(&editor_state_ref.borrow(), block.id), + EditMode::Inactive => false, + }, edit_mode: edit_mode.clone(), allow_write: can_write_sources, }; @@ -2407,6 +2419,27 @@ fn update_pending_line(line: &Element, from: Position, to: Position) -> Result<( mod tests { use super::*; + #[test] + fn source_editor_preserves_yaml_curation_when_rebuilding_target_outputs() { + let target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", "curation": {"enabled": false, "catalog_selection": "curated", "tmdb": {"enabled": false, "api": {"access_token": "test-token"}, "trending": [ + {"kind": "movie", "time_window": "week", "limit": 37, "category_name": "Saved", "create_xtream_category": false}, + {"kind": "tv", "time_window": "day", "limit": 100, "category_name": "Shows"}, + {"kind": "movie", "time_window": "day", "category_name": "Movies"} + ]}}})).unwrap(); + let mut state = EditorState::default(); + state.blocks.push(create_block(1, BlockType::Target, BlockInstance::Target(Rc::new(target.clone())))); + state.blocks.push(create_block(2, BlockType::OutputXtream, create_instance(BlockType::OutputXtream))); + state.connections.push(Connection { from: 1, to: 2 }); + assert!(output_has_target_curation(&state, 2), "disabled declarations still own policy"); + assert!(!output_has_target_curation(&state, 1)); + let result = editor_state_to_sources_config(&SourcesConfigDto::default(), &state); + let saved = &result.sources[0].targets[0]; + assert_eq!(saved.curation, target.curation); + assert!(matches!(&saved.output[0], TargetOutputDto::Xtream(output) if output.trakt.is_none())); + let restored: SourcesConfigDto = serde_json::from_value(serde_json::to_value(&result).unwrap()).unwrap(); + assert_eq!(restored.sources[0].targets[0].curation, target.curation); + } + #[test] fn media_server_blocks_create_matching_input_types() { let cases = [ diff --git a/frontend/src/app/components/source_editor/output_xtream_form.rs b/frontend/src/app/components/source_editor/output_xtream_form.rs index 93ac512b7..d7c9a8fbb 100644 --- a/frontend/src/app/components/source_editor/output_xtream_form.rs +++ b/frontend/src/app/components/source_editor/output_xtream_form.rs @@ -222,16 +222,14 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { let trakt_lists_state = use_state(Vec::::new); let trakt_charts_state = use_state(Vec::::new); - let catalog_selection_options = use_memo(trakt_state.form.catalog_selection, { - let translate = translate.clone(); - move |selection| { + let catalog_selection_options = + use_memo((trakt_state.form.catalog_selection, translate.clone()), |(selection, translate)| { build_options( [TraktCatalogSelection::Full, TraktCatalogSelection::Curated], selection, |value| html! { translate.t(trakt_catalog_selection_label_key(*value)) }, ) - } - }); + }); let show_trakt_list_form_state = use_state(|| false); let show_trakt_chart_form_state = use_state(|| false); let editing_trakt_list_index_state = use_state(|| None::); @@ -451,6 +449,9 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { }; let render_trakt = || { + if source_editor_ctx.output_curation_managed { + return html! {

{translate.t("LABEL.CURATION_YAML_NOTICE")}

 }; + } let trakt_lists = trakt_lists_state.clone(); let trakt_charts = trakt_charts_state.clone(); let trakt_form = trakt_state.clone(); @@ -667,14 +668,16 @@ pub fn XtreamTargetOutputView(props: &XtreamTargetOutputViewProps) -> Html { let trakt_lists = (*trakt_lists_state).clone(); let trakt_charts = (*trakt_charts_state).clone(); - output.trakt = build_trakt_output_config( - trakt_state.data().enabled, - trakt_state.data().catalog_selection, - trakt_state.data().include_xtream_base_categories, - trakt_api_state.data().clone(), - trakt_lists, - trakt_charts, - ); + if !source_editor_ctx.output_curation_managed { + output.trakt = build_trakt_output_config( + trakt_state.data().enabled, + trakt_state.data().catalog_selection, + trakt_state.data().include_xtream_base_categories, + trakt_api_state.data().clone(), + trakt_lists, + trakt_charts, + ); + } source_editor_ctx .on_form_change diff --git a/frontend/src/app/components/source_editor/target_form.rs b/frontend/src/app/components/source_editor/target_form.rs index 043a98a10..6ba1e052e 100644 --- a/frontend/src/app/components/source_editor/target_form.rs +++ b/frontend/src/app/components/source_editor/target_form.rs @@ -14,7 +14,7 @@ use shared::{ error::TuliproxError, model::{ permission::Permission, ClusterFlags, ConfigTargetDto, ConfigTargetFilterDto, ConfigTargetOptions, - ProcessingOrder, TargetBouquetMode, TargetBouquetStatusDto, + CurationCatalogSelection, CurationConfigDto, ProcessingOrder, TargetBouquetMode, TargetBouquetStatusDto, }, utils::Internable, }; @@ -47,6 +47,8 @@ const LABEL_LOWERCASE_XMLTV_DISPLAY_NAMES: &str = "LABEL.LOWERCASE_XMLTV_DISPLAY const LABEL_MAIN: &str = "LABEL.MAIN_CONFIG"; const LABEL_OPTIONS: &str = "LABEL.OPTIONS"; const LABEL_BOUQUETS: &str = "LABEL.BOUQUETS"; +const LABEL_CURATION_CATALOG_SELECTION_FULL: &str = "LABEL.TRAKT_CATALOG_SELECTION_FULL"; +const LABEL_CURATION_CATALOG_SELECTION_CURATED: &str = "LABEL.TRAKT_CATALOG_SELECTION_CURATED"; #[derive(Clone, PartialEq)] enum BouquetStatus { @@ -55,6 +57,21 @@ enum BouquetStatus { Unavailable, } +fn curation_catalog_selection_label_key(selection: CurationCatalogSelection) -> &'static str { + match selection { + CurationCatalogSelection::Full => LABEL_CURATION_CATALOG_SELECTION_FULL, + CurationCatalogSelection::Curated => LABEL_CURATION_CATALOG_SELECTION_CURATED, + } +} + +fn curation_summary(curation: &CurationConfigDto, catalog_selection: &str) -> String { + format!( + "{catalog_selection} · Trakt: {} · TMDB: {}", + curation.trakt.as_ref().map_or(0, |source| source.lists.len() + source.charts.len()), + curation.tmdb.as_ref().map_or(0, |source| source.trending.len()) + ) +} + #[derive(Copy, Clone, PartialEq, Eq)] enum TargetFormPage { Main, @@ -594,6 +611,19 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { let render_edit_mode = || { html! {
+ if let Some(curation) = &target_form_state.form.curation { + +

{translate.t("LABEL.CURATION_YAML_NOTICE")}

+ { config_field_bool!(curation, translate.t(LABEL_ENABLED), enabled) } + { config_field_custom!( + translate.t("LABEL.CURATION"), + curation_summary( + curation, + &translate.t(curation_catalog_selection_label_key(curation.catalog_selection)) + ) + ) } + + }
{render_target()} @@ -664,7 +694,7 @@ pub fn ConfigTargetView(props: &ConfigTargetViewProps) -> Html { #[cfg(test)] mod tests { use super::{ConfigTargetOptionsFormAction, ConfigTargetOptionsFormState}; - use shared::model::ConfigTargetOptions; + use shared::model::{ConfigTargetOptions, CurationCatalogSelection, CurationConfigDto}; use std::rc::Rc; use yew::prelude::Reducible; @@ -672,6 +702,44 @@ mod tests { ConfigTargetOptionsFormState { form: ConfigTargetOptions::default(), modified: false }.into() } + #[test] + fn target_form_name_edits_preserve_yaml_curation() { + let dto: shared::model::ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "before", "curation": {"enabled": false, "tmdb": {"enabled": false, "api": {"access_token": "test-token"}, "trending": [ + {"kind": "movie", "time_window": "week", "limit": 500, "create_xtream_category": false, "category_name": "Saved"}, + {"kind": "tv", "time_window": "day", "limit": 100, "category_name": "Shows"}, + {"kind": "movie", "time_window": "day", "category_name": "Movies"} + ]}}})).unwrap(); + let state = Rc::new(super::ConfigTargetFormState { form: dto.clone(), modified: false }); + let state = state.reduce(super::ConfigTargetFormAction::Name("after".to_string())); + assert_eq!(state.form.curation, dto.curation); + assert_eq!(state.form.name, "after"); + } + + #[test] + fn target_curation_summary_localizes_selection_and_preserves_provider_counts() { + let curation: CurationConfigDto = serde_json::from_value(serde_json::json!({ + "catalog_selection": "curated", + "trakt": { + "lists": [{"user": "alice", "list_slug": "watchlist", "content_type": "vod", "create_xtream_category": false}], + "charts": [{"kind": "movies", "chart": "popular", "create_xtream_category": false}] + }, + "tmdb": { + "trending": [{"kind": "movie", "time_window": "week", "limit": 500, "create_xtream_category": false}] + } + })) + .unwrap(); + + assert_eq!( + super::curation_catalog_selection_label_key(CurationCatalogSelection::Full), + "LABEL.TRAKT_CATALOG_SELECTION_FULL" + ); + assert_eq!( + super::curation_catalog_selection_label_key(curation.catalog_selection), + "LABEL.TRAKT_CATALOG_SELECTION_CURATED" + ); + assert_eq!(super::curation_summary(&curation, "Curated catalog"), "Curated catalog · Trakt: 2 · TMDB: 1"); + } + #[test] fn lowercase_epg_ids_action_updates_nested_option() { let state = default_options_state().reduce(ConfigTargetOptionsFormAction::LowercaseEpgIds(true)); diff --git a/shared/src/model/config/curation.rs b/shared/src/model/config/curation.rs new file mode 100644 index 000000000..dcada51b0 --- /dev/null +++ b/shared/src/model/config/curation.rs @@ -0,0 +1,448 @@ +use crate::{ + defaults::{default_as_true, is_true}, + error::TuliproxError, + model::{TargetOutputDto, TmdbCurationConfigDto, TraktContentType, TraktSourceConfigDto, XtreamCluster}, +}; +use deunicode::deunicode; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use strum_macros::{Display, EnumString}; + +#[derive(Debug, Default, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, Display, EnumString)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase", ascii_case_insensitive)] +pub enum CurationCatalogSelection { + #[default] + Full, + Curated, +} + +impl CurationCatalogSelection { + pub const fn is_full(&self) -> bool { matches!(self, Self::Full) } + + pub const fn is_curated(self) -> bool { matches!(self, Self::Curated) } +} + +/// One target-wide catalog policy with concrete discovery sources. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct CurationConfigDto { + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub enabled: bool, + #[serde(default, skip_serializing_if = "CurationCatalogSelection::is_full")] + pub catalog_selection: CurationCatalogSelection, + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub include_xtream_base_categories: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trakt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tmdb: Option, +} + +impl Default for CurationConfigDto { + fn default() -> Self { + Self { + enabled: true, + catalog_selection: CurationCatalogSelection::Full, + include_xtream_base_categories: true, + trakt: None, + tmdb: None, + } + } +} + +impl CurationConfigDto { + pub fn has_active_selectors(&self) -> bool { + self.enabled + && (self.trakt.as_ref().is_some_and(|source| source.enabled && source.has_selectors()) + || self.tmdb.as_ref().is_some_and(|source| source.enabled && !source.trending.is_empty())) + } + + /// Validate the policy against its outputs without changing declaration ownership. + pub fn prepare(&mut self, outputs: &[TargetOutputDto]) -> Result<(), TuliproxError> { + if outputs.iter().any(|output| matches!(output, TargetOutputDto::Xtream(xtream) if xtream.trakt.is_some())) { + return Err(TuliproxError::Config( + "target.curation and output[].trakt cannot both be configured; choose one declaration".to_string(), + )); + } + if let Some(trakt) = &mut self.trakt { + trakt.prepare(self.enabled)?; + } + if let Some(tmdb) = &mut self.tmdb { + tmdb.prepare(self.enabled)?; + } + if !self.enabled { + return Ok(()); + } + if !self.has_active_selectors() && (!self.catalog_selection.is_full() || !self.include_xtream_base_categories) { + return Err(TuliproxError::Config( + "Enabled curation with non-default policy requires at least one active selector".to_string(), + )); + } + + let trakt = self.trakt.as_ref().filter(|source| source.enabled); + let tmdb = self.tmdb.as_ref().filter(|source| source.enabled); + let projects_categories = trakt.is_some_and(|source| trakt_projections(source).next().is_some()) + || tmdb.is_some_and(|source| source.trending.iter().any(|selector| selector.create_xtream_category)); + let has_xtream = outputs.iter().any(|output| matches!(output, TargetOutputDto::Xtream(_))); + if !has_xtream && (projects_categories || !self.include_xtream_base_categories) { + return Err(TuliproxError::Config( + "Curation category projection and base-category suppression require an Xtream output".to_string(), + )); + } + + // Existing Trakt-only and ordinary base-group collisions retain their behavior. + // Only new TMDB projections reserve names against all other active projections. + let mut names = HashSet::new(); + if let Some(source) = trakt { + for (scope, name) in trakt_projections(source) { + let normalized = normalize_category_name(name); + if matches!(scope, TraktContentType::Vod | TraktContentType::Both) { + names.insert((XtreamCluster::Video, normalized.clone())); + } + if matches!(scope, TraktContentType::Series | TraktContentType::Both) { + names.insert((XtreamCluster::Series, normalized)); + } + } + } + if let Some(source) = tmdb { + for (index, selector) in source.trending.iter().enumerate() { + if !selector.create_xtream_category { + continue; + } + if let Some(name) = &selector.category_name { + let key = (selector.kind.xtream_cluster(), normalize_category_name(name)); + if !names.insert(key) { + return Err(TuliproxError::Config(format!( + "curation.tmdb.trending[{index}].category_name conflicts with another projection in the same media cluster" + ))); + } + } + } + } + Ok(()) + } +} + +fn trakt_projections(source: &TraktSourceConfigDto) -> impl Iterator { + source + .lists + .iter() + .filter(|selector| selector.create_xtream_category) + .filter_map(|selector| selector.category_name.as_deref().map(|name| (selector.content_type, name))) + .chain( + source.charts.iter().filter(|selector| selector.create_xtream_category).filter_map(|selector| { + selector.category_name.as_deref().map(|name| (selector.kind.content_type(), name)) + }), + ) +} + +fn normalize_category_name(name: &str) -> String { deunicode(name.trim()).to_lowercase() } + +pub(super) fn prepare_selector_category( + category_name: &mut Option, + create_xtream_category: bool, + validate: bool, + selector_name: &str, +) -> Result<(), TuliproxError> { + *category_name = category_name.take().map(|name| name.trim().to_string()).filter(|name| !name.is_empty()); + if validate && create_xtream_category && category_name.is_none() { + return Err(TuliproxError::Config(format!( + "{selector_name} category_name is required when create_xtream_category is true" + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{ + M3uTargetOutputDto, TmdbTrendingKind, TmdbTrendingTimeWindow, TraktCatalogSelection, TraktConfigDto, + XtreamTargetOutputDto, + }; + use serde_json::{json, Value}; + + fn xtream() -> Vec { vec![TargetOutputDto::Xtream(XtreamTargetOutputDto::default())] } + + fn m3u() -> Vec { vec![TargetOutputDto::M3u(M3uTargetOutputDto::default())] } + + fn config(value: Value) -> CurationConfigDto { serde_json::from_value(value).expect("curation config shape") } + + fn tmdb_selector() -> Value { json!({"kind": "movie", "time_window": "week", "category_name": "Trending"}) } + + fn mixed() -> Value { + json!({ + "catalog_selection": "curated", + "trakt": {"charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt Popular"}]}, + "tmdb": {"trending": [tmdb_selector()]} + }) + } + + #[test] + fn curation_defaults_preserve_the_full_catalog_and_omit_default_fields() { + let mut dto = config(json!({})); + assert_eq!(dto, CurationConfigDto::default()); + dto.prepare(&m3u()).expect("source-less default policy is a no-op"); + assert_eq!(serde_json::to_value(dto).unwrap(), json!({})); + assert_eq!(TraktCatalogSelection::Full, CurationCatalogSelection::Full); + assert_eq!("CURATED".parse::().unwrap(), CurationCatalogSelection::Curated); + } + + #[test] + fn curation_tmdb_only_requires_no_trakt_or_metadata_credential() { + let mut dto = config(json!({"tmdb": {"trending": [tmdb_selector()]}})); + dto.prepare(&xtream()).expect("missing token is not a config-load failure"); + assert!(dto.trakt.is_none()); + let tmdb = dto.tmdb.unwrap(); + assert!(tmdb.api.access_token.is_empty()); + assert!(tmdb.enabled); + assert_eq!(tmdb.trending[0].kind, TmdbTrendingKind::Movie); + assert_eq!(tmdb.trending[0].time_window, TmdbTrendingTimeWindow::Week); + assert_eq!(tmdb.trending[0].limit, 100); + assert!(tmdb.trending[0].create_xtream_category); + } + + #[test] + fn curation_mixed_sources_round_trip_one_policy() { + let mut dto = config(mixed()); + dto.prepare(&xtream()).unwrap(); + assert_eq!(dto.catalog_selection, CurationCatalogSelection::Curated); + let yaml = serde_saphyr::to_string(&dto).unwrap(); + let mut restored: CurationConfigDto = serde_saphyr::from_str(&yaml).unwrap(); + restored.prepare(&xtream()).unwrap(); + assert_eq!(restored, dto); + let value = serde_json::to_value(dto).unwrap(); + assert!(value["trakt"].get("catalog_selection").is_none()); + assert!(value["tmdb"].get("catalog_selection").is_none()); + } + + #[test] + fn curation_selection_only_supports_m3u_and_preserves_unused_labels() { + let mut value = mixed(); + value["trakt"]["charts"][0]["create_xtream_category"] = json!(false); + value["tmdb"]["trending"][0]["create_xtream_category"] = json!(false); + value["tmdb"]["trending"][0]["category_name"] = json!(" Saved label "); + let mut dto = config(value); + dto.prepare(&m3u()).expect("selection does not require Xtream"); + assert_eq!(dto.tmdb.unwrap().trending[0].category_name.as_deref(), Some("Saved label")); + } + + #[test] + fn curation_rejects_both_owners_even_when_disabled_or_source_less() { + for enabled in [false, true] { + for legacy_enabled in [false, true] { + let mut dto = config(json!({"enabled": enabled})); + let outputs = vec![TargetOutputDto::Xtream(XtreamTargetOutputDto { + trakt: Some(TraktConfigDto { enabled: legacy_enabled, ..TraktConfigDto::default() }), + ..XtreamTargetOutputDto::default() + })]; + let error = dto.prepare(&outputs).expect_err("two declarations must not have implicit precedence"); + let message = error.to_string(); + assert!(message.contains("curation")); + assert!(message.contains("output[].trakt")); + } + } + } + + #[test] + fn curation_non_default_policy_needs_active_selectors() { + for policy in [json!({"catalog_selection": "curated"}), json!({"include_xtream_base_categories": false})] { + for sources in [json!({}), json!({"tmdb": {"enabled": false, "trending": [tmdb_selector()]}})] { + let mut value = policy.clone(); + value.as_object_mut().unwrap().extend(sources.as_object().unwrap().clone()); + let mut dto = config(value); + assert!(dto.prepare(&xtream()).is_err()); + dto.enabled = false; + dto.prepare(&xtream()).expect("disabled policy is retained without execution"); + } + } + } + + #[test] + fn curation_disabled_sources_do_not_invalidate_an_active_sibling() { + let mut value = mixed(); + value["trakt"]["enabled"] = json!(false); + value["trakt"]["charts"][0].as_object_mut().unwrap().remove("category_name"); + let mut dto = config(value); + dto.prepare(&xtream()).expect("incomplete disabled sibling is not required"); + assert!(dto.trakt.unwrap().charts[0].category_name.is_none()); + } + + #[test] + fn curation_disabled_block_preserves_incomplete_category_and_policy_edits() { + let mut value = mixed(); + value["enabled"] = json!(false); + value["include_xtream_base_categories"] = json!(false); + value["tmdb"]["trending"][0]["category_name"] = json!(" "); + value["trakt"]["charts"][0].as_object_mut().unwrap().remove("category_name"); + let mut dto = config(value); + dto.prepare(&m3u()).expect("disabled projection fields may remain incomplete"); + assert_eq!(dto.catalog_selection, CurationCatalogSelection::Curated); + assert!(!dto.include_xtream_base_categories); + assert!(dto.tmdb.unwrap().trending[0].category_name.is_none()); + } + + #[test] + fn curation_active_projection_requires_xtream_for_either_source() { + for value in [ + json!({"tmdb": {"trending": [tmdb_selector()]}}), + json!({"trakt": {"charts": [{"kind": "shows", "chart": "trending", "category_name": "Shows"}]}}), + ] { + let mut dto = config(value); + assert!(dto.prepare(&m3u()).unwrap_err().to_string().contains("Xtream")); + dto.prepare(&xtream()).expect("active projections with Xtream"); + } + } + + #[test] + fn curation_hiding_xtream_base_categories_requires_xtream_even_without_projections() { + let mut selector = tmdb_selector(); + selector["create_xtream_category"] = json!(false); + let mut dto = config(json!({"include_xtream_base_categories": false, "tmdb": {"trending": [selector]}})); + assert!(dto.prepare(&m3u()).is_err()); + dto.prepare(&xtream()).unwrap(); + } + + #[test] + fn curation_projection_name_is_trimmed_and_required_only_for_active_projections() { + for name in [json!(null), json!(""), json!(" \t\n ")] { + let mut selector = tmdb_selector(); + selector["category_name"] = name; + let mut dto = config(json!({"tmdb": {"trending": [selector]}})); + assert!(dto.prepare(&xtream()).unwrap_err().to_string().contains("category_name")); + dto.tmdb.as_mut().unwrap().trending[0].create_xtream_category = false; + dto.prepare(&m3u()).unwrap(); + } + let mut selector = tmdb_selector(); + selector["category_name"] = json!(" Trending "); + let mut dto = config(json!({"tmdb": {"trending": [selector]}})); + dto.prepare(&xtream()).unwrap(); + assert_eq!(dto.tmdb.unwrap().trending[0].category_name.as_deref(), Some("Trending")); + } + + #[test] + fn curation_trakt_and_tmdb_normalized_projection_collisions_are_rejected() { + for (left, right) in [("Résumé", " RESUME "), ("Trending", "trending")] { + let mut value = mixed(); + value["trakt"]["charts"][0]["category_name"] = json!(left); + value["tmdb"]["trending"][0]["category_name"] = json!(right); + let mut dto = config(value); + let message = dto.prepare(&xtream()).unwrap_err().to_string(); + assert!(message.contains("tmdb.trending[0].category_name")); + } + } + + #[test] + fn curation_tmdb_collisions_depend_on_scope_not_current_matches() { + let mut value = json!({"tmdb": {"trending": [tmdb_selector(), tmdb_selector()]}}); + value["tmdb"]["trending"][1]["time_window"] = json!("day"); + assert!(config(value.clone()).prepare(&xtream()).is_err()); + value["tmdb"]["trending"][1]["kind"] = json!("tv"); + config(value).prepare(&xtream()).expect("equal labels in different media clusters are allowed"); + } + + #[test] + fn curation_trakt_both_lists_reserve_names_in_both_clusters() { + for kind in ["movie", "tv"] { + let mut selector = tmdb_selector(); + selector["kind"] = json!(kind); + let mut dto = config(json!({ + "trakt": {"lists": [{"user": "alice", "list_slug": "list", "content_type": "both", "category_name": "Trending"}]}, + "tmdb": {"trending": [selector]} + })); + assert!(dto.prepare(&xtream()).is_err()); + } + } + + #[test] + fn curation_non_projecting_and_disabled_sources_reserve_no_names() { + let mut value = mixed(); + value["trakt"]["charts"][0]["category_name"] = json!("Trending"); + for source in ["trakt", "tmdb"] { + let mut disabled = value.clone(); + disabled[source]["enabled"] = json!(false); + config(disabled).prepare(&xtream()).unwrap(); + } + for (source, selectors) in [("trakt", "charts"), ("tmdb", "trending")] { + let mut selection_only = value.clone(); + selection_only[source][selectors][0]["create_xtream_category"] = json!(false); + config(selection_only).prepare(&xtream()).unwrap(); + } + } + + #[test] + fn curation_preserves_existing_trakt_only_collision_behavior() { + let chart = json!({"kind": "movies", "chart": "trending", "category_name": "Legacy Chart"}); + let mut dto = config(json!({"trakt": {"charts": [chart.clone(), chart]}})); + dto.prepare(&xtream()).expect("do not impose new rules on Trakt-only config"); + assert_eq!(dto.trakt.unwrap().charts.len(), 2); + } + + #[test] + fn curation_child_policy_and_unknown_fields_are_rejected_even_when_disabled() { + for source in ["trakt", "tmdb"] { + for field in ["catalog_selection", "include_xtream_base_categories", "unexpected"] { + let value = json!({"enabled": false, source: {field: "full"}}); + assert!(serde_json::from_value::(value).is_err()); + } + } + assert!(serde_json::from_value::(json!({"unexpected": true})).is_err()); + } + + #[test] + fn curation_tmdb_requires_explicit_kind_and_window_even_when_disabled() { + for field in ["kind", "time_window"] { + let mut selector = tmdb_selector(); + selector.as_object_mut().unwrap().remove(field); + let value = json!({"enabled": false, "tmdb": {"trending": [selector]}}); + assert!(serde_json::from_value::(value).is_err(), "missing {field}"); + } + for (field, invalid) in [("kind", "all"), ("time_window", "month")] { + let mut selector = tmdb_selector(); + selector[field] = json!(invalid); + let value = json!({"enabled": false, "tmdb": {"trending": [selector]}}); + assert!(serde_json::from_value::(value).is_err(), "unsupported {field}"); + } + } + + #[test] + fn curation_tmdb_accepts_all_four_bounded_feed_shapes() { + for kind in ["movie", "tv"] { + for window in ["day", "week"] { + let value = json!({"tmdb": {"trending": [{"kind": kind, "time_window": window, "create_xtream_category": false}]}}); + let mut dto = config(value); + dto.prepare(&m3u()).unwrap(); + } + } + } + + #[test] + fn curation_tmdb_debug_redacts_tokens_without_losing_serialized_credentials() { + let secret = "private-test-token"; + let mut dto = + config(json!({"tmdb": {"api": {"access_token": format!(" {secret} ")}, "trending": [tmdb_selector()]}})); + dto.prepare(&xtream()).unwrap(); + assert!(!format!("{dto:?}").contains(secret)); + assert_eq!(dto.tmdb.as_ref().unwrap().api.access_token, secret); + assert_eq!(serde_json::to_value(&dto).unwrap()["tmdb"]["api"]["access_token"], secret); + dto.tmdb.as_mut().unwrap().trending[0].category_name = None; + assert!(!dto.prepare(&xtream()).unwrap_err().to_string().contains(secret)); + } + + #[test] + fn curation_tmdb_header_validity_belongs_to_runtime_not_source_loading() { + for token in ["", " ", "invalid\nheader"] { + let mut dto = config(json!({"tmdb": {"api": {"access_token": token}, "trending": [tmdb_selector()]}})); + dto.prepare(&xtream()).expect("credential availability must not invalidate the containing source config"); + } + } + + #[test] + fn curation_tmdb_does_not_accept_metadata_or_arbitrary_endpoint_options() { + for field in ["api_key", "url", "language", "page", "limit"] { + let value = json!({"tmdb": {"api": {field: "not-a-supported-option"}}}); + assert!(serde_json::from_value::(value).is_err(), "unsupported API field {field}"); + } + } +} diff --git a/shared/src/model/config/mod.rs b/shared/src/model/config/mod.rs index 241e746f5..050968347 100644 --- a/shared/src/model/config/mod.rs +++ b/shared/src/model/config/mod.rs @@ -6,6 +6,7 @@ mod base; mod byte_size; mod cache; mod config_type; +mod curation; mod epg; mod epg_smart_match; mod geoip; @@ -41,6 +42,7 @@ mod stream_history; mod target; mod template_definition; mod time_units; +mod tmdb_curation; mod trakt; mod video_download; mod web_auth; @@ -63,6 +65,7 @@ pub use base::*; pub use byte_size::*; pub use cache::*; pub use config_type::*; +pub use curation::*; pub use disk_alert::*; pub use epg::*; pub use epg_smart_match::*; @@ -104,6 +107,7 @@ pub use stream_history::*; pub use target::*; pub use template_definition::*; pub use time_units::*; +pub use tmdb_curation::*; pub use trakt::*; pub use video_download::*; pub use web_auth::*; diff --git a/shared/src/model/config/target.rs b/shared/src/model/config/target.rs index 69e07b114..37f05a035 100644 --- a/shared/src/model/config/target.rs +++ b/shared/src/model/config/target.rs @@ -6,8 +6,8 @@ use crate::{ error::TuliproxError, foundation::{get_filter, Filter}, model::{ - ClusterFlags, ConfigFavouritesDto, ConfigRenameDto, ConfigSortDto, HdHomeRunDeviceOverview, PatternTemplate, - Prepare, PrepareAll, ProcessingOrder, StrmExportStyle, TargetType, TraktConfigDto, + ClusterFlags, ConfigFavouritesDto, ConfigRenameDto, ConfigSortDto, CurationConfigDto, HdHomeRunDeviceOverview, + PatternTemplate, Prepare, PrepareAll, ProcessingOrder, StrmExportStyle, TargetType, TraktConfigDto, }, utils::is_blank_optional_string, }; @@ -417,6 +417,8 @@ pub struct ConfigTargetDto { #[serde(default)] pub output: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] + pub curation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub rename: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub mapping: Option>, @@ -440,6 +442,7 @@ impl Default for ConfigTargetDto { sort: None, filter: ConfigTargetFilterDto::default(), output: Vec::new(), + curation: None, rename: None, mapping: None, favourites: None, @@ -467,6 +470,10 @@ impl ConfigTargetDto { return Err(TuliproxError::ConfigTarget("target name required".to_string())); } + if let Some(curation) = &mut self.curation { + curation.prepare(&self.output)?; + } + let mut m3u_cnt = 0; let mut xtream_cnt = 0; let mut strm_cnt = 0; @@ -660,6 +667,76 @@ mod tests { fn xtream_output() -> TargetOutputDto { TargetOutputDto::Xtream(XtreamTargetOutputDto::default()) } + #[test] + fn shipped_tmdb_example_is_safe_by_default_and_prepares_when_explicitly_enabled() { + let sources: crate::model::SourcesConfigDto = + serde_saphyr::from_str(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../config/source.yml"))).unwrap(); + let mut target = sources + .sources + .into_iter() + .flat_map(|source| source.targets) + .find(|target| target.name == "tmdb-discovery-example") + .expect("shipped TMDB example"); + assert!(!target.enabled); + assert!(!target.curation.as_ref().unwrap().enabled); + assert!(target.curation.as_ref().unwrap().tmdb.as_ref().unwrap().api.access_token.is_empty()); + target.prepare(1, None, None).unwrap(); + target.enabled = true; + target.curation.as_mut().unwrap().enabled = true; + target.prepare(1, None, None).unwrap(); + let yaml = serde_saphyr::to_string(&target).unwrap(); + let restored: ConfigTargetDto = serde_saphyr::from_str(&yaml).unwrap(); + assert_eq!(restored.curation, target.curation); + } + + #[test] + fn target_curation_prepares_and_round_trips_tmdb_only_and_mixed_sources() { + for with_trakt in [false, true] { + let mut value = serde_json::json!({"name": "discovery", "output": [{"type": "xtream"}, {"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "movie", "time_window": "week", "limit": 37, "category_name": "TMDB"}]}}}); + if with_trakt { + value["curation"]["trakt"] = + serde_json::json!({"charts": [{"kind": "movies", "chart": "popular", "category_name": "Trakt"}]}); + } + let mut target: ConfigTargetDto = serde_json::from_value(value).unwrap(); + target.prepare(1, None, None).unwrap(); + let restored: ConfigTargetDto = serde_saphyr::from_str(&serde_saphyr::to_string(&target).unwrap()).unwrap(); + assert_eq!(restored.curation, target.curation); + assert!(matches!(&restored.output[0], TargetOutputDto::Xtream(output) if output.trakt.is_none())); + } + } + + #[test] + fn target_curation_rejects_dual_declarations_before_preparing_legacy_values() { + let mut target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", + "curation": {"enabled": false}, "output": [{"type": "xtream", "trakt": {"catalog_selection": "curated"}}]})) + .unwrap(); + assert!(target.prepare(1, None, None).unwrap_err().to_string().contains("output[].trakt")); + } + + #[test] + fn target_curation_m3u_only_requires_selection_only_entries() { + let mut target: ConfigTargetDto = serde_json::from_value(serde_json::json!({"name": "discovery", "output": [{"type": "m3u"}], + "curation": {"catalog_selection": "curated", "tmdb": {"trending": [{"kind": "tv", "time_window": "day", "limit": 500, "category_name": "Shows"}]}}})).unwrap(); + assert!(target.prepare(1, None, None).is_err()); + target.curation.as_mut().unwrap().tmdb.as_mut().unwrap().trending[0].create_xtream_category = false; + target.prepare(1, None, None).unwrap(); + } + + #[test] + fn target_curation_absence_and_null_keep_the_legacy_surface() { + for value in [ + serde_json::json!({"name": "discovery", "output": [{"type": "xtream", "trakt": {}}]}), + serde_json::json!({"name": "discovery", "curation": null, "output": [{"type": "xtream", "trakt": {}}]}), + ] { + let mut target: ConfigTargetDto = serde_json::from_value(value).unwrap(); + target.prepare(1, None, None).unwrap(); + let serialized = serde_json::to_value(target).unwrap(); + assert!(serialized.get("curation").is_none()); + assert!(serialized["output"][0].get("trakt").is_some()); + } + } + #[test] fn target_filter_string_roundtrips_as_string() { let dto = serde_saphyr::from_str::( diff --git a/shared/src/model/config/tmdb_curation.rs b/shared/src/model/config/tmdb_curation.rs new file mode 100644 index 000000000..162849ed5 --- /dev/null +++ b/shared/src/model/config/tmdb_curation.rs @@ -0,0 +1,207 @@ +use super::curation::prepare_selector_category; +use crate::{ + defaults::{default_as_true, is_true}, + error::TuliproxError, + model::XtreamCluster, +}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// TMDB application Read Access Token. Serialization is lossless; diagnostics are not. +#[derive(Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TmdbCurationApiConfigDto { + #[serde(default)] + pub access_token: String, +} + +impl fmt::Debug for TmdbCurationApiConfigDto { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TmdbCurationApiConfigDto").field("access_token", &"[redacted]").finish() + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum TmdbTrendingKind { + Movie, + Tv, +} + +impl TmdbTrendingKind { + pub const fn xtream_cluster(self) -> XtreamCluster { + match self { + Self::Movie => XtreamCluster::Video, + Self::Tv => XtreamCluster::Series, + } + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum TmdbTrendingTimeWindow { + Day, + Week, +} + +pub const TMDB_TRENDING_DEFAULT_LIMIT: u32 = 100; +pub const TMDB_TRENDING_MAX_LIMIT: u32 = 500; + +const fn default_trending_limit() -> u32 { TMDB_TRENDING_DEFAULT_LIMIT } +const fn is_default_trending_limit(limit: &u32) -> bool { *limit == TMDB_TRENDING_DEFAULT_LIMIT } +pub const fn is_valid_tmdb_trending_limit(limit: u32) -> bool { limit >= 1 && limit <= TMDB_TRENDING_MAX_LIMIT } + +// deserialize_u32 permits quoted numbers in our YAML loader. Inspect the scalar type +// instead so YAML and JSON have the same integer-only contract (including disabled drafts). +fn deserialize_trending_limit<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + struct Integer; + impl serde::de::Visitor<'_> for Integer { + type Value = u32; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("an unsigned integer") } + fn visit_u64(self, value: u64) -> Result { + u32::try_from(value).map_err(|_| E::custom("TMDB trending limit exceeds u32")) + } + fn visit_i64(self, value: i64) -> Result { + u32::try_from(value).map_err(|_| E::custom("TMDB trending limit must be unsigned and fit u32")) + } + } + deserializer.deserialize_any(Integer) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TmdbTrendingConfigDto { + pub kind: TmdbTrendingKind, + pub time_window: TmdbTrendingTimeWindow, + /// Unique remote references before local matching, not a desired number of matches. + #[serde( + default = "default_trending_limit", + deserialize_with = "deserialize_trending_limit", + skip_serializing_if = "is_default_trending_limit" + )] + pub limit: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub category_name: Option, + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub create_xtream_category: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TmdbCurationConfigDto { + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub enabled: bool, + #[serde(default)] + pub api: TmdbCurationApiConfigDto, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub trending: Vec, +} + +impl Default for TmdbCurationConfigDto { + fn default() -> Self { Self { enabled: true, api: TmdbCurationApiConfigDto::default(), trending: Vec::new() } } +} + +#[cfg(test)] +mod tests { + use crate::model::ConfigTargetDto; + + fn target(selector_fields: &str, target_enabled: bool, curation_enabled: bool, source_enabled: bool) -> String { + format!( + r#"{{"name":"discovery","enabled":{target_enabled},"output":[{{"type":"xtream"}}], + "curation":{{"enabled":{curation_enabled},"tmdb":{{"enabled":{source_enabled},"trending":[ + {{"kind":"movie","time_window":"week","category_name":"Movies"{selector_fields}}}]}}}}}}"# + ) + } + + #[test] + fn curation_tmdb_item_limit_defaults_and_boundaries_round_trip() { + for (fields, limit) in [("", 100), (",\"limit\":100", 100), (",\"limit\":1", 1), (",\"limit\":500", 500)] { + for yaml in [false, true] { + let document = target(fields, true, true, true); + let mut dto: ConfigTargetDto = if yaml { + serde_saphyr::from_str(&document).unwrap() + } else { + serde_json::from_str(&document).unwrap() + }; + dto.prepare(1, None, None).unwrap(); + let value = serde_json::to_value(&dto).unwrap(); + let selector = &value["curation"]["tmdb"]["trending"][0]; + if limit == 100 { + assert!(selector.get("limit").is_none()); + } else { + assert_eq!(selector["limit"], limit); + } + let restored: ConfigTargetDto = + serde_saphyr::from_str(&serde_saphyr::to_string(&dto).unwrap()).unwrap(); + assert_eq!(restored.curation, dto.curation); + } + } + } + + #[test] + fn curation_tmdb_item_limit_rejects_types_and_ranges_even_when_disabled() { + for raw in [ + "null", + "0", + "-1", + "501", + "4294967295", + "4294967296", + "18446744073709551616", + "1.5", + "100.0", + "true", + "\"100\"", + "[]", + "{}", + ] { + for switches in [(true, true, true), (false, true, true), (true, false, true), (true, true, false)] { + let document = target(&format!(",\"limit\":{raw}"), switches.0, switches.1, switches.2); + for mut parsed in [ + serde_json::from_str::(&document).ok(), + serde_saphyr::from_str::(&document).ok(), + ] { + if let Some(dto) = &mut parsed { + assert!(dto.prepare(1, None, None).is_err(), "accepted {raw}, switches {switches:?}"); + } + } + } + } + } + + #[test] + fn curation_tmdb_item_limit_rejects_scope_and_paging_fields_even_when_disabled() { + for fields in [ + ",\"scope\":\"first_page\"", + ",\"scope\":null", + ",\"scope\":\"first_page\",\"limit\":100", + ",\"page\":1", + ",\"max_pages\":5", + ] { + for enabled in [false, true] { + let document = target(fields, enabled, enabled, enabled); + assert!(serde_json::from_str::(&document).is_err(), "{fields}"); + assert!(serde_saphyr::from_str::(&document).is_err(), "{fields}"); + } + } + } +} + +impl TmdbCurationConfigDto { + pub(super) fn prepare(&mut self, curation_enabled: bool) -> Result<(), TuliproxError> { + self.api.access_token = self.api.access_token.trim().to_string(); + for selector in &mut self.trending { + // Even disabled editor drafts must retain a valid acquisition bound. + if !is_valid_tmdb_trending_limit(selector.limit) { + return Err(TuliproxError::Config("TMDB trending limit must be an integer in 1..=500".to_string())); + } + prepare_selector_category( + &mut selector.category_name, + selector.create_xtream_category, + curation_enabled && self.enabled, + "TMDB trending", + )?; + } + Ok(()) + } +} diff --git a/shared/src/model/config/trakt.rs b/shared/src/model/config/trakt.rs index b9234a1b9..33a47d737 100644 --- a/shared/src/model/config/trakt.rs +++ b/shared/src/model/config/trakt.rs @@ -1,3 +1,4 @@ +use super::curation::prepare_selector_category; use crate::{ defaults::{ default_as_true, default_trakt_fuzzy_threshold, is_false, is_true, DEFAULT_USER_AGENT, TRAKT_API_URL, @@ -33,20 +34,8 @@ pub struct TraktApiConfigDto { pub user_agent: String, } -#[derive(Debug, Default, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, Display, EnumString)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase", ascii_case_insensitive)] -pub enum TraktCatalogSelection { - #[default] - Full, - Curated, -} - -impl TraktCatalogSelection { - pub const fn is_full(&self) -> bool { matches!(self, Self::Full) } - - pub const fn is_curated(self) -> bool { matches!(self, Self::Curated) } -} +// Keep the existing Rust/public configuration vocabulary compatible. +pub use super::curation::CurationCatalogSelection as TraktCatalogSelection; impl TraktApiConfigDto { pub fn prepare(&mut self) { @@ -150,6 +139,35 @@ impl Default for TraktListConfigDto { } } +/// Trakt source settings under a target-owned curation policy. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TraktSourceConfigDto { + #[serde(default = "default_as_true", skip_serializing_if = "is_true")] + pub enabled: bool, + #[serde(default)] + pub api: TraktApiConfigDto, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub lists: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub charts: Vec, +} + +impl Default for TraktSourceConfigDto { + fn default() -> Self { + Self { enabled: true, api: TraktApiConfigDto::default(), lists: Vec::new(), charts: Vec::new() } + } +} + +impl TraktSourceConfigDto { + pub fn has_selectors(&self) -> bool { !self.lists.is_empty() || !self.charts.is_empty() } + + pub(super) fn prepare(&mut self, curation_enabled: bool) -> Result<(), TuliproxError> { + self.api.prepare(); + prepare_trakt_selectors(&mut self.lists, &mut self.charts, curation_enabled && self.enabled) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct TraktConfigDto { @@ -180,17 +198,26 @@ impl Default for TraktConfigDto { } } -fn prepare_selector_category( - category_name: &mut Option, - create_xtream_category: bool, +fn prepare_trakt_selectors( + lists: &mut [TraktListConfigDto], + charts: &mut [TraktChartConfigDto], validate: bool, - selector_name: &str, ) -> Result<(), TuliproxError> { - *category_name = category_name.take().map(|name| name.trim().to_string()).filter(|name| !name.is_empty()); - if validate && create_xtream_category && category_name.is_none() { - return Err(TuliproxError::Config(format!( - "Trakt {selector_name} category_name is required when create_xtream_category is true" - ))); + for selector in lists { + prepare_selector_category( + &mut selector.category_name, + selector.create_xtream_category, + validate, + "Trakt list", + )?; + } + for selector in charts { + prepare_selector_category( + &mut selector.category_name, + selector.create_xtream_category, + validate, + "Trakt chart", + )?; } Ok(()) } @@ -209,22 +236,7 @@ impl TraktConfigDto { pub fn prepare(&mut self) -> Result<(), TuliproxError> { self.api.prepare(); - for selector in &mut self.lists { - prepare_selector_category( - &mut selector.category_name, - selector.create_xtream_category, - self.enabled, - "list", - )?; - } - for selector in &mut self.charts { - prepare_selector_category( - &mut selector.category_name, - selector.create_xtream_category, - self.enabled, - "chart", - )?; - } + prepare_trakt_selectors(&mut self.lists, &mut self.charts, self.enabled)?; if self.enabled && !self.has_selectors() && !self.is_source_less_noop() { return Err(TuliproxError::Config( "Enabled Trakt curation with non-default policy requires at least one list or chart".to_string(),