From f2370b1545e5d2017169291371b3fcb93a4ea31b Mon Sep 17 00:00:00 2001 From: Stenio Araujo Date: Fri, 21 Aug 2026 13:23:53 -0400 Subject: [PATCH] Support VOD (movies) and Series from M3U sources in the Xtream API (#830) * feat(parser): support tvg-type VOD/Series in M3U and synthesize SeriesInfo - recognize tvg-type=movie|vod|video as Xtream VOD, series|episode as Series, and live explicitly, taking precedence over extension inference - for series rows, treat group-title as the show name and #EXTGRP only as the default Xtream series category, keeping @Group mappable - group series rows by category + show name and synthesize SeriesStreamProperties/SeriesInfo via the existing Xtream series path - extract SxxEyy from titles; generate deterministic u32 episode IDs; use numeric tvg-id as series ID when available - build docker with rust 1.95 (project requires rustc >= 1.95.0) * fix(xtream): serve embedded seasons/episodes for M3U-synthesized SeriesInfo M3U-synthesized SeriesInfo items carry their series details in additional_properties but were not recognized by the stream info endpoint (item_type is not local, no provider URL), so get_series_info returned an empty array and IPTV clients showed no seasons/episodes. Route items with embedded details through the local info-document path so embedded seasons/episodes are serialized by to_info_document. * fix(parser): normalize series category in M3U series-map key --- backend/src/api/endpoints/xtream_api.rs | 11 +- .../src/api/model/recording/recording_disk.rs | 23 +- backend/src/processing/parser/m3u.rs | 395 +++++++++++++++++- backend/src/processing/parser/stalker.rs | 14 +- backend/src/processing/processor/library.rs | 8 +- docker/Dockerfile | 4 +- shared/src/defaults/epg.rs | 7 +- shared/src/model/playlist.rs | 5 + shared/src/model/stream_properties.rs | 14 +- shared/src/utils/constants.rs | 25 +- shared/src/utils/hash_utils.rs | 172 ++++++++ 11 files changed, 620 insertions(+), 58 deletions(-) diff --git a/backend/src/api/endpoints/xtream_api.rs b/backend/src/api/endpoints/xtream_api.rs index f13ff82fb..811f85ff9 100644 --- a/backend/src/api/endpoints/xtream_api.rs +++ b/backend/src/api/endpoints/xtream_api.rs @@ -1288,8 +1288,9 @@ pub async fn xtream_get_stream_info_response( let input = app_state.app_config.get_input_by_name(&pli.input_name); let is_media_server = input.as_ref().is_some_and(|i| i.input_type.is_media_server()); - // handle local items and media server - if pli.item_type.is_local() || is_media_server { + // handle local items, media server, and items with embedded details + // (e.g. M3U-synthesized SeriesInfo carrying seasons/episodes). + if pli.item_type.is_local() || is_media_server || pli.has_details() { let Some(xtream_output) = target.get_xtream_output() else { return empty_stream_info_response(cluster); }; @@ -1537,6 +1538,12 @@ async fn xtream_get_catchup_response( xtream_get_item_for_stream_id(req_virtual_id, app_state, target, Some(XtreamCluster::Live)).await ); + // Content filter: hidden items expose no catch-up table either, so a + // plan-tier-restricted user cannot probe hidden channels via catchup. + if !(user.t_filter.is_none() || user.allows_content(&shared::model::PlaylistItem::from(&pli))) { + return axum::Json(json!(ShortEpgResultDto::default())).into_response(); + } + let input = try_option_bad_request!(app_state.app_config.get_input_by_name(&pli.input_name)); let mut info_url = try_option_bad_request!(xtream::get_xtream_player_api_action_url( diff --git a/backend/src/api/model/recording/recording_disk.rs b/backend/src/api/model/recording/recording_disk.rs index 6078ac0d5..231c3c41f 100644 --- a/backend/src/api/model/recording/recording_disk.rs +++ b/backend/src/api/model/recording/recording_disk.rs @@ -120,20 +120,21 @@ pub fn filesystem_capacity_for(path: &Path) -> Option<(u64, u64)> { { let cstr = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).ok()?; let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; + // SAFETY: `cstr` is a valid NUL-terminated C string; `&raw mut stat` // is a writable pointer to a zeroed struct. let rc = unsafe { libc::statvfs(cstr.as_ptr(), &raw mut stat) }; if rc != 0 { return None; } - #[cfg(target_pointer_width = "32")] - let bsize = u64::from(stat.f_frsize); - #[cfg(target_pointer_width = "64")] - let bsize = stat.f_frsize; - let total = stat.f_blocks.saturating_mul(bsize); + + let bsize = stat.f_frsize as u64; + let total = (stat.f_blocks as u64).saturating_mul(bsize); + // `f_bavail`, not `f_bfree`: the service user cannot use the // root-reserved blocks, so counting them would understate pressure. - let available = stat.f_bavail.saturating_mul(bsize); + let available = (stat.f_bavail as u64).saturating_mul(bsize); + Some((total, available)) } #[cfg(windows)] @@ -177,20 +178,20 @@ pub fn free_bytes_for(path: &Path) -> Option { { let cstr = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).ok()?; let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; + // SAFETY: `cstr` is a valid NUL-terminated C string; `&raw mut stat` // is a writable pointer to a zeroed struct. let rc = unsafe { libc::statvfs(cstr.as_ptr(), &raw mut stat) }; if rc != 0 { return None; } - #[cfg(target_pointer_width = "32")] - let bsize = u64::from(stat.f_frsize); - #[cfg(target_pointer_width = "64")] - let bsize = stat.f_frsize; + + let bsize = stat.f_frsize as u64; + // `f_bavail` is the bytes available to a non-privileged // caller — the worker runs as the service user and is // subject to the same reservation as any other user. - Some(stat.f_bavail.saturating_mul(bsize)) + Some((stat.f_bavail as u64).saturating_mul(bsize)) } #[cfg(windows)] { diff --git a/backend/src/processing/parser/m3u.rs b/backend/src/processing/parser/m3u.rs index 33684f9c3..383ccc9fa 100644 --- a/backend/src/processing/parser/m3u.rs +++ b/backend/src/processing/parser/m3u.rs @@ -1,17 +1,42 @@ use crate::model::{Config, ConfigInput}; use crate::utils::request::DynReader; +use regex::Regex; use shared::model::{ CatchupAttribute, CatchupProperties, LiveStreamProperties, PlaylistGroup, PlaylistItem, PlaylistItemHeader, - PlaylistItemType, StreamProperties, XtreamCluster, + PlaylistItemType, SeriesStreamDetailEpisodeProperties, SeriesStreamDetailProperties, + SeriesStreamDetailSeasonProperties, SeriesStreamProperties, StreamProperties, XtreamCluster, }; -use shared::utils::{extract_id_from_url, extract_numeric_id_from_url, Internable}; -use shared::defaults::{default_supported_video_extensions}; +use shared::utils::{ + extract_id_from_url, extract_numeric_id_from_url, fnv1a_32_parts, get_provider_id, parse_season_episode, + Internable, CONSTANTS, +}; +use shared::defaults::default_supported_video_extensions; use std::borrow::BorrowMut; +use std::collections::HashMap; use std::sync::Arc; use tokio::io::AsyncBufReadExt; use indexmap::IndexMap; use crate::repository::CategoryKey; +/// User-configured episode pattern if set, else `CONSTANTS.re_episode_code` +/// (`SxxEyy` / `NxNN` / "Season x Episode y" / "Episode y", compiled once at startup). +/// The default is wrapped in `Arc` once via `LazyLock` so repeated +/// `build_series_info` calls do not re-clone the regex. +fn resolve_episode_pattern(cfg: &Config) -> Arc { + cfg.video + .as_ref() + .and_then(|v| v.download.as_ref()) + .and_then(|d| d.episode_pattern.as_ref().map(Arc::clone)) + .unwrap_or_else(default_episode_pattern_arc) +} + +fn default_episode_pattern_arc() -> Arc { + static DEFAULT: std::sync::OnceLock> = std::sync::OnceLock::new(); + DEFAULT + .get_or_init(|| Arc::new(CONSTANTS.re_episode_code.clone())) + .clone() +} + // other implementations like calculating text_distance on all titles took too much time // we keep it now as simple as possible and less memory intensive. fn get_title_group(text: &Arc) -> Arc { @@ -126,6 +151,7 @@ enum M3uToken { TvgChno, GroupTitle, TvgId, + TvgType, TvgName, TvgLogo, TvgLogoSmall, @@ -158,6 +184,7 @@ fn classify_possible_id(bytes: &[u8]) -> M3uToken { } } +#[allow(clippy::too_many_lines)] fn classify_token(t: &str) -> M3uToken { let bytes = t.as_bytes(); @@ -188,13 +215,15 @@ fn classify_token(t: &str) -> M3uToken { classify_possible_id(bytes) } } - 8 => { + 8 => { if eq_ascii(bytes, b"tvg-chno") { M3uToken::TvgChno } else if eq_ascii(bytes, b"tvg-name") { M3uToken::TvgName } else if eq_ascii(bytes, b"tvg-logo") { M3uToken::TvgLogo + } else if eq_ascii(bytes, b"tvg-type") { + M3uToken::TvgType } else { classify_possible_id(bytes) } @@ -261,6 +290,15 @@ fn classify_token(t: &str) -> M3uToken { } } +fn parse_declared_type(value: &str) -> Option<(XtreamCluster, PlaylistItemType)> { + match value.trim().to_ascii_lowercase().as_str() { + "movie" | "vod" | "video" => Some((XtreamCluster::Video, PlaylistItemType::Video)), + "series" | "episode" => Some((XtreamCluster::Series, PlaylistItemType::Series)), + "live" => Some((XtreamCluster::Live, PlaylistItemType::Live)), + _ => None, + } +} + fn ensure_live_stream_properties(header: &mut PlaylistItemHeader) -> &mut LiveStreamProperties { if !matches!(header.additional_properties.as_ref(), Some(StreamProperties::Live(_))) { header.additional_properties = Some(StreamProperties::Live(Box::default())); @@ -318,16 +356,13 @@ fn process_header_internal( url: String, default_catchup_correction: Option<&Arc>, ) -> PlaylistItemHeader { - let url_types = if video_suffixes.iter().any(|suffix| url.ends_with(suffix)) { - // TODO find Series based on group or configured names - Some((XtreamCluster::Video, PlaylistItemType::Video)) - } else { - None - }; + let extension_type = video_suffixes.iter().any(|suffix| url.ends_with(suffix)) + .then_some((XtreamCluster::Video, PlaylistItemType::Video)); let mut plih = create_empty_playlistitem_header(input_name, url); let mut it = content.chars(); let mut stack = String::with_capacity(64); + let mut declared_type = None; let is_extinf = token_till(&mut stack, &mut it, ':', false) .is_some_and(|off| stack[off..].eq_ignore_ascii_case("#EXTINF")); stack.clear(); @@ -358,6 +393,7 @@ fn process_header_internal( } M3uToken::TvgChno => plih.chno = stack[val_off..].parse::().unwrap_or(0), M3uToken::GroupTitle => plih.group = stack[val_off..].intern(), + M3uToken::TvgType => declared_type = parse_declared_type(&stack[val_off..]), M3uToken::TvgId => plih.epg_channel_id = if stack.len() == val_off { None } else { Some(stack[val_off..].intern()) }, M3uToken::TvgName => plih.name = stack[val_off..].intern(), M3uToken::TvgLogo => plih.logo = stack[val_off..].intern(), @@ -411,9 +447,9 @@ fn process_header_internal( } apply_catchup_properties(&mut plih, catchup, default_catchup_correction); } - if let Some((url_cluster, url_item_type)) = url_types { - plih.xtream_cluster = url_cluster; - plih.item_type = url_item_type; + if let Some((cluster, item_type)) = declared_type.or(extension_type) { + plih.xtream_cluster = cluster; + plih.item_type = item_type; } { @@ -528,7 +564,18 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp header.upstream_user_agent = upstream_user_agent.take(); header.source_ordinal = ord_counter; ord_counter += 1; - if header.group.is_empty() { + if header.xtream_cluster.is_series() { + let series_name = if header.group.is_empty() { + get_title_group(&header.title) + } else { + header.group.clone() + }; + header.parent_code = series_name; + header.group = group_value + .as_deref() + .filter(|value| !value.trim().is_empty()) + .map_or_else(|| "Series".intern(), Internable::intern); + } else if header.group.is_empty() { if let Some(group_value) = group_value { header.group = group_value; } else { @@ -540,10 +587,120 @@ pub async fn consume_m3u(cfg: &Config, input: &ConfigInp } } +fn build_series_info(cfg: &Config, items: Vec) -> Option { + let episode_pattern = resolve_episode_pattern(cfg); + let first = items.first()?; + let series_name = first.header.parent_code.clone(); + let default_category = first.header.group.clone(); + let series_id = get_provider_id( + first.header.epg_channel_id.as_deref().unwrap_or(""), + &first.header.url, + ) + .unwrap_or_else(|| fnv1a_32_parts(&[first.header.input_name.as_ref(), first.header.parent_code.as_ref()])); + let source_ordinal = first.header.source_ordinal; + let logo = first.header.logo.clone(); + let input_name = first.header.input_name.clone(); + + let mut episodes = Vec::with_capacity(items.len()); + let mut season_counts: HashMap = HashMap::new(); + let mut dropped = 0_usize; + let item_count = items.len(); + + for item in items { + let Some((season, episode_num)) = parse_season_episode(&item.header.title, &episode_pattern) else { + dropped += 1; + continue; + }; + + *season_counts.entry(season).or_insert(0) += 1; + episodes.push(SeriesStreamDetailEpisodeProperties { + id: fnv1a_32_parts(&[item.header.input_name.as_ref(), item.header.url.as_ref()]), + episode_num, + season, + title: item.header.title.clone(), + direct_source: item.header.url.clone(), + movie_image: item.header.logo.clone(), + ..Default::default() + }); + } + + if episodes.is_empty() { + if dropped > 0 { + log::debug!( + "m3u series: dropped all {dropped} items in group {series_name:?} (no SxxEyy token)", + ); + } + return None; + } + if dropped > 0 { + log::debug!( + "m3u series: dropped {dropped} of {item_count} items in group {series_name:?} (no SxxEyy token)", + ); + } + + episodes.sort_by_key(|episode| (episode.season, episode.episode_num)); + + let mut seasons: Vec = season_counts + .into_iter() + .map(|(season_number, episode_count)| SeriesStreamDetailSeasonProperties { + name: format!("Season {season_number}").intern(), + season_number, + episode_count, + overview: None, + air_date: None, + cover: Some(logo.clone()), + cover_tmdb: None, + cover_big: None, + duration: None, + }) + .collect(); + seasons.sort_by_key(|season| season.season_number); + + let properties = SeriesStreamProperties { + name: series_name.clone(), + series_id, + cover: logo.clone(), + details: Some(SeriesStreamDetailProperties::new(None, seasons, Some(episodes))), + ..SeriesStreamProperties::default() + }; + + Some(PlaylistItem { + header: PlaylistItemHeader { + id: series_id.to_string().intern(), + name: series_name.clone(), + logo, + group: default_category, + title: series_name, + url: "".intern(), + item_type: PlaylistItemType::SeriesInfo, + xtream_cluster: XtreamCluster::Series, + additional_properties: Some(StreamProperties::Series(Box::new(properties))), + source_ordinal, + input_name, + ..PlaylistItemHeader::default() + }, + }) +} + pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> Vec { let mut group_map: IndexMap> = IndexMap::new(); + let mut series_map: IndexMap<(Arc, Arc), Vec> = IndexMap::new(); + consume_m3u(cfg, input, lines, |item| { + if item.header.xtream_cluster.is_series() { + let key = ( + shared::utils::deunicode_string(&item.header.group) + .to_lowercase() + .intern(), + shared::utils::deunicode_string(&item.header.parent_code) + .to_lowercase() + .intern(), + ); + series_map.entry(key).or_default().push(item); + return; + } + let key = { let header = &item.header; let normalized_group = shared::utils::deunicode_string(&header.group).to_lowercase().intern(); @@ -552,6 +709,18 @@ pub async fn parse_m3u(cfg: &Config, input: &ConfigInput, lines: DynReader) -> V group_map.entry(key).or_default().push(item); }).await; + for ((_category, _series_name), items) in series_map { + if let Some(series_info) = build_series_info(cfg, items) { + let normalized_group = shared::utils::deunicode_string(&series_info.header.group) + .to_lowercase() + .intern(); + group_map + .entry((XtreamCluster::Series, normalized_group)) + .or_default() + .push(series_info); + } + } + let mut grp_id = 0; group_map.into_values().filter_map(|channels| { // create a group based on the first playlist item @@ -571,10 +740,11 @@ mod test { use crate::model::{Config, ConfigInput}; use crate::utils::request::DynReader; use shared::{ - model::StreamProperties, - utils::Internable, + model::{PlaylistItemType, REGEX_CACHE, StreamProperties, XtreamCluster}, + utils::{fnv1a_32, parse_season_episode, Internable, CONSTANTS}, }; - use crate::processing::parser::m3u::{classify_token, process_header, M3uToken}; + use shared::defaults::default_episode_pattern; + use crate::processing::parser::m3u::{classify_token, parse_m3u, process_header, M3uToken}; use tokio::io::AsyncWriteExt; fn make_reader(content: &str) -> DynReader { @@ -623,6 +793,197 @@ mod test { assert!(matches!(classify_token("foo"), M3uToken::Unknown)); } + #[test] + fn test_tvg_type_movie_without_extension_is_vod() { + let input = "movies".intern(); + let video_suffixes = vec!["mp4".to_string(), "mkv".to_string()]; + let url = "https://example.test/movie/user/pass/ea8c49de0be27dfa4f2ee47d8b10d4f7"; + let line = r#"#EXTINF:0 tvg-type="movie" tvg-id="tt37619362" group-title="Movie VOD",Example Movie Name (2025)"#; + + let item = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(item.xtream_cluster, XtreamCluster::Video); + assert_eq!(item.item_type, PlaylistItemType::Video); + } + + #[test] + fn test_tvg_type_series_without_extension_is_series() { + let input = "series".intern(); + let video_suffixes = vec!["mp4".to_string(), "mkv".to_string()]; + let url = "https://example.test/series/user/pass/1214b7c13c8e318d695a6f2a33ac580d"; + let line = r#"#EXTINF:0 tvg-type="series" tvg-id="156988" group-title="Example Show Name",Example Show Name S02E05"#; + + let item = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(item.xtream_cluster, XtreamCluster::Series); + assert_eq!(item.item_type, PlaylistItemType::Series); + } + + #[test] + fn test_explicit_series_beats_video_extension() { + let input = "series".intern(); + let video_suffixes = vec!["mp4".to_string()]; + let url = "https://example.test/series/user/pass/episode.mp4"; + let line = r#"#EXTINF:0 tvg-type="series" tvg-id="156988" group-title="Example Show Name",Example Show Name S02E05"#; + + let item = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(item.xtream_cluster, XtreamCluster::Series); + assert_eq!(item.item_type, PlaylistItemType::Series); + } + + #[test] + fn test_explicit_live_beats_video_extension() { + let input = "live".intern(); + let video_suffixes = vec!["ts".to_string()]; + let url = "https://example.test/live/user/pass/channel.ts"; + let line = r#"#EXTINF:0 tvg-type="live" group-title="News",Channel"#; + + let item = process_header(&input, &video_suffixes, line, url.to_string()); + assert_eq!(item.xtream_cluster, XtreamCluster::Live); + assert_eq!(item.item_type, PlaylistItemType::Live); + } + + #[test] + fn test_parse_episode_code() { + let pattern = REGEX_CACHE + .get_or_compile(&default_episode_pattern().unwrap_or_default()) + .expect("default episode pattern must compile"); + assert_eq!(parse_season_episode("Show S02E05 [1080p]", &pattern), Some((2, 5))); + assert_eq!(parse_season_episode("Show s8e2", &pattern), Some((8, 2))); + assert_eq!(parse_season_episode("Show without episode", &pattern), None); + // Module-level static is identical to the one we just built. + assert_eq!( + parse_season_episode("Show S02E05", &CONSTANTS.re_episode_code), + Some((2, 5)), + ); + } + + #[test] + fn test_fnv1a_32_is_deterministic() { + // Lock the contract: two parses of the same M3U produce + // identical series/episode IDs, so virtual-id persistence does + // not drift across reloads. + let a = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); + let b = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); + assert_eq!(a, b); + // Different inputs must collide to different IDs (smoke test + // only — collision rates are tested separately). + assert_ne!(a, fnv1a_32("input_a:Show Name:https://example.test/series/ep2")); + assert_ne!(a, fnv1a_32("input_b:Show Name:https://example.test/series/ep1")); + // Hash must never be zero (sentinel value). + assert!(fnv1a_32("") > 0); + } + + #[tokio::test] + async fn test_series_group_drops_episodes_without_sxxeyy() { + // Pin the silent-drop behaviour: an episode whose title has + // no SxxEyy token is excluded from the synthesised series + // item, while the SxxEyy-tagged sibling survives. + let content = r#"#EXTM3U +#EXTINF:0 tvg-type="series" tvg-id="156988" tvg-logo="https://example.test/poster.jpg" group-title="Example Show Name",Example Show Name S02E05 +https://example.test/series/user/pass/s02e05hash +#EXTINF:0 tvg-type="series" tvg-id="156988" tvg-logo="https://example.test/poster.jpg" group-title="Example Show Name",Example Show Name Pilot +https://example.test/series/user/pass/pilothash +"#; + + let groups = parse_m3u(&Config::default(), &test_input(), make_reader(content)).await; + + let series = groups + .iter() + .flat_map(|group| group.channels.iter()) + .find(|item| item.header.xtream_cluster.is_series()) + .expect("expected one synthesised series item"); + let episodes = match series + .header + .additional_properties + .as_ref() + .expect("series properties present") + { + shared::model::StreamProperties::Series(s) => s + .details + .as_ref() + .and_then(|d| d.episodes.as_ref()) + .expect("series episodes present"), + _ => panic!("expected Series properties"), + }; + assert_eq!(episodes.len(), 1, "the Pilot title has no SxxEyy and is dropped"); + assert_eq!(episodes[0].episode_num, 5); + assert_eq!(episodes[0].season, 2); + } + + #[tokio::test] + async fn test_series_keeps_name_separate_from_mappable_group() { + let content = r#"#EXTM3U +#EXTINF:0 tvg-type="series" tvg-id="156988" tvg-logo="https://example.test/poster.jpg" group-title="Example Show Name",Example Show Name S02E05 +#EXTGRP:TV VOD +https://example.test/series/user/pass/episodehash +"#; + + let groups = parse_m3u(&Config::default(), &test_input(), make_reader(content)).await; + + assert_eq!(groups.len(), 1); + let series = &groups[0].channels[0]; + assert_eq!(series.header.xtream_cluster, XtreamCluster::Series); + assert_eq!(series.header.item_type, PlaylistItemType::SeriesInfo); + assert_eq!(&*series.header.name, "Example Show Name"); + assert_eq!(&*series.header.group, "TV VOD"); // default; mapping.yml may replace @Group + assert_eq!(&*series.header.input_name, "input"); + assert!(series.header.additional_properties.as_ref().is_some_and(StreamProperties::has_details)); + } + + #[tokio::test] + async fn test_series_embeds_seasons_and_episodes_for_info_endpoint() { + let content = r#"#EXTM3U +#EXTINF:0 tvg-type="series" tvg-id="156988" tvg-logo="https://example.test/poster.jpg" group-title="Example Show Name",Example Show Name S02E05 +#EXTGRP:TV VOD +https://example.test/series/user/pass/episode-1 +#EXTINF:0 tvg-type="series" tvg-id="156988" group-title="Example Show Name",Example Show Name S02E06 +#EXTGRP:TV VOD +https://example.test/series/user/pass/episode-2 +#EXTINF:0 tvg-type="series" tvg-id="156988" group-title="Example Show Name",Example Show Name S01E01 +#EXTGRP:TV VOD +https://example.test/series/user/pass/episode-3 +"#; + + let groups = parse_m3u(&Config::default(), &test_input(), make_reader(content)).await; + + assert_eq!(groups.len(), 1); + let series = &groups[0].channels[0]; + let Some(StreamProperties::Series(props)) = series.header.additional_properties.as_ref() else { + panic!("expected series properties"); + }; + let details = props.details.as_ref().expect("series details"); + let episodes = details.episodes.as_ref().expect("episodes"); + let seasons = details.seasons.as_ref().expect("seasons"); + assert_eq!(episodes.len(), 3); + assert_eq!(seasons.len(), 2); + assert_eq!((episodes[0].season, episodes[0].episode_num), (1, 1)); + assert_eq!(episodes[0].direct_source.as_ref(), "https://example.test/series/user/pass/episode-3"); + } + + #[tokio::test] + async fn test_series_groups_normalize_case_and_diacritics() { + let content = r#"#EXTM3U +#EXTINF:0 tvg-type="series" tvg-id="156988" group-title="Show",Show S02E05 +#EXTGRP:TV ÉPISODES +https://example.test/series/user/pass/episode-1 +#EXTINF:0 tvg-type="series" tvg-id="156988" group-title="Show",Show S02E06 +#EXTGRP:tv episodes +https://example.test/series/user/pass/episode-2 +"#; + + let groups = parse_m3u(&Config::default(), &test_input(), make_reader(content)).await; + + assert_eq!(groups.len(), 1); + let series = &groups[0].channels[0]; + assert_eq!(series.header.xtream_cluster, XtreamCluster::Series); + assert_eq!(series.header.item_type, PlaylistItemType::SeriesInfo); + assert_eq!(&*series.header.name, "Show"); + let Some(StreamProperties::Series(props)) = series.header.additional_properties.as_ref() else { + panic!("expected series properties"); + }; + let details = props.details.as_ref().expect("series details"); + assert_eq!(details.episodes.as_ref().map(Vec::len), Some(2)); + } + #[test] fn test_process_header_1() { let input = "hello".intern(); diff --git a/backend/src/processing/parser/stalker.rs b/backend/src/processing/parser/stalker.rs index 3c5c4e5b4..f747e5845 100644 --- a/backend/src/processing/parser/stalker.rs +++ b/backend/src/processing/parser/stalker.rs @@ -17,7 +17,7 @@ use shared::model::stalker::{ StalkerCommandVariantDto, StalkerPlaybackDescriptorDto, StalkerPlaybackMode, StalkerStreamKind, }; use shared::model::stalker_item::StalkerPlaylistItem; -use shared::utils::Internable; +use shared::utils::{fnv1a_32, stable_episode_storage_id, Internable}; use std::collections::HashSet; use std::sync::Arc; @@ -419,18 +419,6 @@ fn map_stalker_episode( } } -fn stable_episode_storage_id(series_id: u32, season_number: u32, episode_id: &str, episode_number: u32) -> u32 { - let key = format!("{series_id}:{season_number}:{episode_id}:{episode_number}"); - fnv1a_32(&key) -} - -fn fnv1a_32(key: &str) -> u32 { - let hash = key.bytes().fold(2_166_136_261_u32, |hash, byte| { - (hash ^ u32::from(byte)).wrapping_mul(16_777_619) - }); - hash.max(1) -} - /// Collision-safe variant of [`stable_episode_storage_id`]: when the hashed id is /// already taken within the current snapshot batch, the key string is deterministically /// re-salted (`:`) and re-hashed until a free slot is found. The snapshot diff --git a/backend/src/processing/processor/library.rs b/backend/src/processing/processor/library.rs index 12e12b257..1e7dd6fe8 100644 --- a/backend/src/processing/processor/library.rs +++ b/backend/src/processing/processor/library.rs @@ -478,11 +478,11 @@ pub fn metadata_cache_entry_to_xtream_series_info( release_date: release_date.map(Into::into), youtube_trailer: youtube_trailer.into(), tmdb: series.tmdb_id, - details: Some(SeriesStreamDetailProperties { - year: series.year, - seasons: Some(seasons), + details: Some(SeriesStreamDetailProperties::new( + series.year, + seasons, episodes, - }), + )), }; Some(StreamProperties::Series(Box::new(properties))) diff --git a/docker/Dockerfile b/docker/Dockerfile index 07b2f4354..32e290f33 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,7 +13,7 @@ FROM mwader/static-ffmpeg:7.1 AS ffmpeg-static # ----------------------------------------------------------------- # Stage 1: Build the Rust binary for production # ----------------------------------------------------------------- -FROM rust@sha256:e8e2bb5ff27ad3b369a4f667392464e6ec399cfe81c1230ae78edb1036b9bd74 AS rust-build +FROM rust:1.95 AS rust-build # Get target architecture from build argument ARG RUST_TARGET=x86_64-unknown-linux-musl @@ -51,7 +51,7 @@ RUN cargo build -p tuliprox --target ${RUST_TARGET} --release # ----------------------------------------------------------------- # Stage 2: Build the rust frontend # ----------------------------------------------------------------- -FROM rust@sha256:e8e2bb5ff27ad3b369a4f667392464e6ec399cfe81c1230ae78edb1036b9bd74 AS trunk-build +FROM rust:1.95 AS trunk-build # Install dependencies for Trunk and WebAssembly RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ diff --git a/shared/src/defaults/epg.rs b/shared/src/defaults/epg.rs index 90f6b6943..0b6c544a8 100644 --- a/shared/src/defaults/epg.rs +++ b/shared/src/defaults/epg.rs @@ -84,10 +84,9 @@ pub fn is_default_epg_name_prefix_separator(v: &Option>) -> bool { current == &default_separator } -pub const DEFAULT_EPISODE_PATTERN: &str = r".*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*"; - -pub fn default_episode_pattern() -> Option { Some(DEFAULT_EPISODE_PATTERN.to_string()) } +pub fn default_episode_pattern() -> Option { Some(crate::utils::constants::EPISODE_PATTERN.to_string()) } pub fn is_blank_or_default_episode_pattern(path: &Option) -> bool { - path.as_ref().is_none_or(|value| value.trim().is_empty() || value.trim() == DEFAULT_EPISODE_PATTERN) + path.as_ref() + .is_none_or(|value| value.trim().is_empty() || value.trim() == crate::utils::constants::EPISODE_PATTERN) } diff --git a/shared/src/model/playlist.rs b/shared/src/model/playlist.rs index e1a848b0e..d6392056b 100644 --- a/shared/src/model/playlist.rs +++ b/shared/src/model/playlist.rs @@ -40,6 +40,11 @@ pub enum XtreamCluster { impl XtreamCluster { pub fn as_str(&self) -> &str { self.as_ref() } + /// True when this cluster is the Xtream `Series` cluster. Used in + /// bucket-key computations and dispatch sites that previously + /// spelled out `== XtreamCluster::Series` inline. + pub fn is_series(self) -> bool { matches!(self, Self::Series) } + pub fn as_stream_type(&self) -> &str { match self { Self::Live => "live", diff --git a/shared/src/model/stream_properties.rs b/shared/src/model/stream_properties.rs index 8ba391b06..96b2a8183 100644 --- a/shared/src/model/stream_properties.rs +++ b/shared/src/model/stream_properties.rs @@ -336,7 +336,7 @@ pub struct SeriesStreamDetailSeasonProperties { pub duration: Option>, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] pub struct SeriesStreamDetailEpisodeProperties { #[serde(default, deserialize_with = "deserialize_number_from_string_or_zero")] pub id: u32, @@ -398,6 +398,18 @@ pub struct SeriesStreamDetailProperties { pub episodes: Option>, } +impl SeriesStreamDetailProperties { + /// Builds a `SeriesStreamDetailProperties`. Empty `seasons` is normalized to + /// `None` so downstream code can short-circuit; `episodes` is kept as-is. + pub fn new( + year: Option, + seasons: Vec, + episodes: Option>, + ) -> Self { + Self { year, seasons: if seasons.is_empty() { None } else { Some(seasons) }, episodes } + } +} + #[derive(Default, Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct SeriesStreamProperties { #[serde(default, deserialize_with = "arc_str_none_default_on_null")] diff --git a/shared/src/utils/constants.rs b/shared/src/utils/constants.rs index a957a8dbf..c3df380bb 100644 --- a/shared/src/utils/constants.rs +++ b/shared/src/utils/constants.rs @@ -93,6 +93,23 @@ pub struct Constants { pub re_html_tag: Regex, } +/// Source regex for `re_episode_code`. Single source of truth: the +/// user-facing config default in `crate::defaults::DEFAULT_EPISODE_PATTERN` +/// is just a reference to this constant. Covers the four historical +/// shapes (compact `SxxEyy` / `NxNN`, verbose `Season X Episode Y`, +/// bare `Episode Y`). Rust regex requires unique capture names across +/// the whole pattern, so the season and episode captures are suffixed +/// per branch (`_s`, `_n`, `_v`, `_vo`/`_vf`); `parse_season_episode` +/// recognises all of them. +pub const EPISODE_PATTERN: &str = concat!( + r"(?i)\b(?:", + r"[Ss](?P\d{1,2})[Ee](?P\d{1,3})", + r"|(?P\d{1,2})x(?P\d{1,3})", + r"|season[\s._-]*(?P\d{1,2})(?:[\s._-]*episode[\s._-]*(?P\d{1,3}))?", + r"|episode[\s._-]*(?P\d{1,3})", + r")\b", +); + pub static CONSTANTS: LazyLock = LazyLock::new(|| { Constants { re_credentials: Regex::new(r"((username|password|token|api_key)=)[^&]*").unwrap(), @@ -147,10 +164,10 @@ pub static CONSTANTS: LazyLock = LazyLock::new(|| { re_quality: Regex::new(r"(?i)\b(x265|4K|UHD|8K|2160p?|1080p?|720p?|480p?|BLURAY|HDTV|DVDRIP|BRRIP|CAM|TS|HDR|DV|SDR)\b").unwrap(), re_classifier_year: Regex::new(r"[\(\[]?(\d{4})[\)\]]?").unwrap(), re_classifier_cleanup: Regex::new(r"(?i)[\s\._-]*(?:s\d+e\d+|\d+x\d+|season[\s\._-]*\d+|episode[\s\._-]*\d+).*$").unwrap(), - re_episode_code: Regex::new( - r"(?i)\b(?:s\d{1,2}e\d{1,3}|\d{1,2}x\d{1,3}|season[\s._-]*\d{1,2}(?:[\s._-]*episode[\s._-]*\d{1,3})?|episode[\s._-]*\d{1,3})\b" - ) - .unwrap(), + // Single source of truth: `EPISODE_PATTERN` (above) is reused + // as the user-facing config default by + // `crate::defaults::DEFAULT_EPISODE_PATTERN`. + re_episode_code: Regex::new(EPISODE_PATTERN).expect("episode pattern must compile"), re_classifier_episode: Regex::new(r"(?i)(?:e|episode|x)[\s\._-]*(\d+)").unwrap(), re_classifier_season: Regex::new(r"(?i)(?:s|season)[\s\._-]*(\d+)").unwrap(), re_classifier_moviedb_id: Regex::new(r"(?i)\b(tmdb|tvdb|imdb)[\s._=-]?(\d+)\b").unwrap(), diff --git a/shared/src/utils/hash_utils.rs b/shared/src/utils/hash_utils.rs index 10bec4c04..97f99c53a 100644 --- a/shared/src/utils/hash_utils.rs +++ b/shared/src/utils/hash_utils.rs @@ -210,9 +210,104 @@ pub fn create_alias_uuid(base_uuid: &UUIDType, mapping_id: &str) -> UUIDType { UUIDType(hasher.finalize().into()) } +// === FNV-1a 32-bit hash for stable numeric IDs. === +// +// Used when an upstream id is missing (e.g. M3U episodes, Stalker synthetic +// storage ids). Deterministic across runs; collisions are extremely rare but +// callers should still de-duplicate within a snapshot batch. + +pub fn fnv1a_32(key: &str) -> u32 { + let hash = key.bytes().fold(2_166_136_261_u32, |hash, byte| (hash ^ u32::from(byte)).wrapping_mul(16_777_619)); + hash.max(1) +} + +/// Streaming FNV-1a over a sequence of string parts joined by `:` without +/// allocating an intermediate `String`. Equivalent to +/// `fnv1a_32(&parts.join(":"))` for any non-empty `parts`, with an empty +/// separator byte between consecutive parts. +pub fn fnv1a_32_parts(parts: &[&str]) -> u32 { + let mut hash = 2_166_136_261_u32; + for (i, part) in parts.iter().enumerate() { + if i > 0 { + hash ^= u32::from(b':'); + hash = hash.wrapping_mul(16_777_619); + } + for &byte in part.as_bytes() { + hash ^= u32::from(byte); + hash = hash.wrapping_mul(16_777_619); + } + } + hash.max(1) +} + +pub fn stable_episode_storage_id(series_id: u32, season_number: u32, episode_id: &str, episode_number: u32) -> u32 { + let key = format!("{series_id}:{season_number}:{episode_id}:{episode_number}"); + fnv1a_32(&key) +} + +// === Season/episode extraction from a configured regex. === +// +// The regex must expose named `season` and `episode` captures. Supported +// shapes include the compact `SxxEyy` / `NxNN` forms and the verbose +// `Season X Episode Y` / `Episode Y` forms. When the regex matches a +// shape without an explicit season (e.g. bare `Episode 5`), the season +// defaults to `1` so the caller still receives a usable pair. +// +// `crate::defaults::default_episode_pattern` exposes `EPISODE_PATTERN` +// as the user-facing default. + +pub fn parse_season_episode(title: &str, pattern: ®ex::Regex) -> Option<(u32, u32)> { + let caps = pattern.captures(title)?; + // The CONSTANTS pattern uses branch-specific capture names + // (`season_s`/`episode_s`, `season_n`/`episode_n`, + // `season_v`/`episode_vf`/`episode_vo`); the simpler DEFAULT + // pattern uses `season`/`episode`. Whichever branch matched, + // pick the first populated capture. + let episode_str = caps + .name("episode") + .or_else(|| caps.name("episode_s")) + .or_else(|| caps.name("episode_n")) + .or_else(|| caps.name("episode_vf")) + .or_else(|| caps.name("episode_vo"))? + .as_str() + .trim(); + let season_str = caps + .name("season") + .or_else(|| caps.name("season_s")) + .or_else(|| caps.name("season_n")) + .or_else(|| caps.name("season_v")) + .map(|m| m.as_str().trim()); + + // New-format patterns expose season and episode as separate + // numeric captures. For legacy user-configured patterns that + // still wrap `SxxEyy` inside a single named capture (e.g. + // `(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2})`), fall back to + // splitting the captured string. + let (season, episode) = match (season_str, episode_str.parse::()) { + (Some(s), Ok(e)) => (s.parse::().ok()?, e), + (None, Ok(e)) => (1, e), + (_, Err(_)) => parse_sxxeyy_fallback(episode_str)?, + }; + Some((season, episode)) +} + +/// Legacy fallback: a user-configured pattern may still expose the +/// whole `SxxEyy` token under one named capture. Strip the leading +/// `S`/`s`, find the next letter, and parse the digit runs around it. +fn parse_sxxeyy_fallback(token: &str) -> Option<(u32, u32)> { + let after_s = token.strip_prefix(|c: char| c.is_ascii_alphabetic()).unwrap_or(token); + let e_idx = after_s.find(|c: char| c.is_ascii_alphabetic())?; + let (s_str, e_str) = after_s.split_at(e_idx); + let season = s_str.trim().parse().ok()?; + let episode = e_str[1..].trim().parse().ok()?; + Some((season, episode)) +} + #[cfg(test)] mod tests { use super::*; + use crate::utils::{constants::EPISODE_PATTERN, CONSTANTS}; + use regex::Regex; // ── extract_id_from_url ──────────────────────────────────────────── @@ -696,4 +791,81 @@ mod tests { ); } } + + #[test] + fn fnv1a_32_is_deterministic_and_nonzero() { + let a = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); + let b = fnv1a_32("input_a:Show Name:https://example.test/series/ep1"); + assert_eq!(a, b); + assert!(fnv1a_32("") > 0); + assert!(fnv1a_32("0") > 0); + assert_ne!(a, fnv1a_32("input_a:Show Name:https://example.test/series/ep2")); + } + + #[test] + fn fnv1a_32_matches_known_vector() { + // FNV-1a 32-bit of "" is 0x811c9dc5 (offset basis). Our implementation + // forces the result to `>= 1` so the sentinel-zero guard kicks in. + assert_eq!(fnv1a_32(""), 0x811c9dc5); + } + + #[test] + fn fnv1a_32_parts_matches_joined_string() { + let joined = fnv1a_32("a:b:c"); + let parts = fnv1a_32_parts(&["a", "b", "c"]); + assert_eq!(joined, parts); + // Empty parts list still produces the offset-basis sentinel. + assert_eq!(fnv1a_32_parts(&[]), fnv1a_32("")); + // Single part produces no separator byte. + assert_eq!(fnv1a_32_parts(&["only"]), fnv1a_32("only")); + } + + #[test] + fn stable_episode_storage_id_keys_separate_contexts() { + // Same (series, season, episode_num, episode_id) tuple must hash + // identically regardless of how the caller order is. + let a = stable_episode_storage_id(1, 1, "10", 1); + let b = stable_episode_storage_id(1, 1, "10", 1); + assert_eq!(a, b); + // Different episode_num must not collide. + assert_ne!(a, stable_episode_storage_id(1, 1, "10", 2)); + // Different series must not collide. + assert_ne!(a, stable_episode_storage_id(2, 1, "10", 1)); + } + + #[test] + fn parse_season_episode_extracts_with_default_pattern() { + let pattern = Regex::new(EPISODE_PATTERN).unwrap(); + assert_eq!(parse_season_episode("Show S02E05 [1080p]", &pattern), Some((2, 5))); + assert_eq!(parse_season_episode("Show s8e2", &pattern), Some((8, 2))); + assert_eq!(parse_season_episode("Show without episode", &pattern), None); + // Pattern only matches the digits grouped after S and E. + assert_eq!(parse_season_episode("Pilot", &pattern), None); + } + + #[test] + fn parse_season_episode_legacy_user_pattern_with_wrapped_episode_capture() { + // A user-configured pattern that still wraps the whole + // `SxxEyy` token inside a single named capture (the + // pre-refactor format) must keep working — the function + // falls back to splitting the captured string. + let legacy = Regex::new(r".*(?P[Ss]\d{1,2}(.*?)[Ee]\d{1,2}).*").unwrap(); + assert_eq!(parse_season_episode("Show S02E05 [1080p]", &legacy), Some((2, 5))); + assert_eq!(parse_season_episode("Show s8e2", &legacy), Some((8, 2))); + } + + #[test] + fn parse_season_episode_handles_verbose_and_fallback_shapes() { + // The CONSTANTS regex covers the four historical shapes. + let pattern = Regex::new(CONSTANTS.re_episode_code.as_str()).unwrap(); + // Verbose `Season X Episode Y` + assert_eq!(parse_season_episode("Show - Season 2 Episode 5", &pattern), Some((2, 5)),); + assert_eq!(parse_season_episode("Show Season 02 Episode 05", &pattern), Some((2, 5)),); + // `NxNN` + assert_eq!(parse_season_episode("Show 1x05", &pattern), Some((1, 5))); + // Bare `Episode Y` falls back to season 1. + assert_eq!(parse_season_episode("Show Episode 7", &pattern), Some((1, 7))); + // No match → None. + assert_eq!(parse_season_episode("Pilot", &pattern), None); + } }