From a027e808b1c0b14a80fc67e96653dea30ce66d8e Mon Sep 17 00:00:00 2001 From: euzu Date: Mon, 30 Dec 2024 10:27:25 +0100 Subject: [PATCH] Resolve Series name processing added --- src/model/playlist.rs | 4 +- src/processing/xtream_parser.rs | 4 +- src/processing/xtream_processor_series.rs | 82 +++++++++-------------- src/repository/kodi_repository.rs | 46 ++++++++----- test/rest-api.http | 10 ++- 5 files changed, 71 insertions(+), 75 deletions(-) diff --git a/src/model/playlist.rs b/src/model/playlist.rs index c745f91c0..6a2d9cbdd 100644 --- a/src/model/playlist.rs +++ b/src/model/playlist.rs @@ -386,7 +386,7 @@ impl PlaylistItem { M3uPlaylistItem { virtual_id: header.virtual_id, provider_id: Rc::clone(&header.id), - name: Rc::clone(&header.name), + name: Rc::clone(if header.item_type == PlaylistItemType::Series {&header.title} else {&header.name}), chno: Rc::clone(&header.chno), logo: Rc::clone(&header.logo), logo_small: Rc::clone(&header.logo_small), @@ -409,7 +409,7 @@ impl PlaylistItem { XtreamPlaylistItem { virtual_id: header.virtual_id, provider_id, - name: Rc::clone(&header.name), + name: Rc::clone(if header.item_type == PlaylistItemType::Series {&header.title} else {&header.name}), logo: Rc::clone(&header.logo), logo_small: Rc::clone(&header.logo_small), group: Rc::clone(&header.group), diff --git a/src/processing/xtream_parser.rs b/src/processing/xtream_parser.rs index cb9ba7eea..a55d85746 100644 --- a/src/processing/xtream_parser.rs +++ b/src/processing/xtream_parser.rs @@ -39,7 +39,7 @@ fn create_xtream_series_info_url(url: &str, username: &str, password: &str, epis } } -pub fn parse_xtream_series_info(info: &Value, group_title: &str, input: &ConfigInput) -> Result>, M3uFilterError> { +pub fn parse_xtream_series_info(info: &Value, group_title: &str, series_name: &str, input: &ConfigInput) -> Result>, M3uFilterError> { let url = input.url.as_str(); let username = input.username.as_ref().map_or("", |v| v); let password = input.password.as_ref().map_or("", |v| v); @@ -53,7 +53,7 @@ pub fn parse_xtream_series_info(info: &Value, group_title: &str, input: &ConfigI header: RefCell::new(PlaylistItemHeader { id: Rc::new(episode.id.to_string()), uuid: Rc::new(hash_string(&episode_url)), - name: Rc::new(episode.title.clone()), + name: Rc::new(series_name.to_string()), logo: Rc::new(episode.info.movie_image.clone()), group: Rc::new(group_title.to_string()), title: Rc::new(episode.title.clone()), diff --git a/src/processing/xtream_processor_series.rs b/src/processing/xtream_processor_series.rs index 7d34a182c..ba315a3af 100644 --- a/src/processing/xtream_processor_series.rs +++ b/src/processing/xtream_processor_series.rs @@ -11,6 +11,8 @@ use crate::{create_resolve_options_function_for_xtream_target, handle_error, han use std::collections::HashMap; use std::fs::File; use std::io::{BufWriter, Write}; +use std::time::Instant; +use log::info; use crate::model::xtream::{XtreamSeriesEpisode, XtreamSeriesInfoEpisode}; const TAG_SERIES_INFO_LAST_MODIFIED: &str = "last_modified"; @@ -51,26 +53,32 @@ fn should_update_series_info(pli: &PlaylistItem, processed_provider_ids: &HashMa } async fn playlist_resolve_series_info(cfg: &Config, errors: &mut Vec, - processed_fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { - let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, processed_fpl).await; + fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { + let mut processed_info_ids = read_processed_series_info_ids(cfg, errors, fpl).await; // we cant write to the indexed-document directly because of the write lock and time-consuming operation. // All readers would be waiting for the lock and the app would be unresponsive. // We collect the content into a wal file and write it once we collected everything. - let Some((wal_content_file, wal_record_file, wal_content_path, wal_record_path)) = create_resolve_info_wal_files(cfg, processed_fpl.input, XtreamCluster::Series) + let Some((wal_content_file, wal_record_file, wal_content_path, wal_record_path)) = create_resolve_info_wal_files(cfg, fpl.input, XtreamCluster::Series) else { return !processed_info_ids.is_empty(); }; let mut content_writer = BufWriter::new(&wal_content_file); let mut record_writer = BufWriter::new(&wal_record_file); let mut content_updated = false; - for pli in processed_fpl.playlistgroups.iter() + let series_info_iter = fpl.playlistgroups.iter() .filter(|&plg| plg.xtream_cluster == XtreamCluster::Series) .flat_map(|plg| &plg.channels) - .filter(|&pli| pli.header.borrow().item_type == PlaylistItemType::SeriesInfo) - { + .filter(|&pli| pli.header.borrow().item_type == PlaylistItemType::SeriesInfo); + + let series_info_count = series_info_iter.clone().count(); + + info!("Found {series_info_count} series info to resolve"); + let start_time = Instant::now(); + let mut processed_series_info = 0; + for pli in series_info_iter { let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids); if should_update { - if let Some(content) = playlist_resolve_download_playlist_item(pli, processed_fpl.input, errors, resolve_delay, XtreamCluster::Series).await { + if let Some(content) = playlist_resolve_download_playlist_item(pli, fpl.input, errors, resolve_delay, XtreamCluster::Series).await { handle_error_and_return!(write_info_content_to_wal_file(&mut content_writer, provider_id, &content), |err| errors.push(notify_err!(format!("Failed to resolve series, could not write to content wal file {err}")))); processed_info_ids.insert(provider_id, ts); @@ -79,7 +87,13 @@ async fn playlist_resolve_series_info(cfg: &Config, errors: &mut Vec 0 && elapsed % 5 == 0 { + info!("resolved {processed_series_info}/{series_info_count} series info"); + } } + info!("resolved {processed_series_info}/{series_info_count} series info"); // content_wal contains the provider_id and series_info with episode listing // record_wal contains provider_id and timestamp if content_updated { @@ -91,9 +105,9 @@ async fn playlist_resolve_series_info(cfg: &Config, errors: &mut Vec, + fpl: &mut FetchedPlaylist<'_>, errors: &mut Vec, ) -> Vec { let mut result: Vec = vec![]; - let input = provider_fpl.input; + let input = fpl.input; let Ok(Some((info_path, idx_path))) = get_input_storage_path(input, &cfg.working_dir) .map(|storage_path| xtream_get_info_file_paths(&storage_path, XtreamCluster::Series)) @@ -131,7 +145,7 @@ async fn process_series_info( }; let mut wal_writer = BufWriter::new(&wal_file); - for plg in provider_fpl + for plg in fpl .playlistgroups .iter_mut() .filter(|plg| plg.xtream_cluster == XtreamCluster::Series) @@ -147,7 +161,11 @@ async fn process_series_info( let Ok(content) = info_reader.get(&provider_id) else { continue; }; match serde_json::from_str::(&content) { Ok(series_content) => { - match parse_xtream_series_info(&series_content, pli.header.borrow().group.as_str(), input) { + let (group, series_name) = { + let header = pli.header.borrow(); + (header.group.clone(), if header.name.is_empty() {header.title.clone()} else { header.name.clone()}) + }; + match parse_xtream_series_info(&series_content, &group, &series_name, input) { Ok(Some(series)) => { for (episode, pli_episode) in &series { let Some(provider_id) = &pli_episode.header.borrow_mut().get_provider_id() else { continue; }; @@ -213,41 +231,3 @@ pub async fn playlist_resolve_series(cfg: &Config, target: &ConfigTarget, processed_fpl.update_playlist(plg); } } - -// -// pub(in crate::processing) async fn playlist_resolve_series(target: &ConfigTarget, errors: &mut Vec, -// pipe: &ProcessingPipe, -// provider_fpl: &mut FetchedPlaylist<'_>, -// processed_fpl: &mut FetchedPlaylist<'_>) { -// let (resolve_series, resolve_series_delay) = -// if let Some(options) = &target.options { -// (options.xtream_resolve_series && provider_fpl.input.input_type == InputType::Xtream && target.has_output(&TargetType::M3u), -// options.xtream_resolve_series_delay) -// } else { -// (false, 0) -// }; -// if resolve_series { -// // collect all series in the processed lists -// let to_process_uuids: HashSet> = processed_fpl.playlistgroups.iter() -// .filter(|plg| plg.xtream_cluster == XtreamCluster::Series) -// .flat_map(|plg| &plg.channels) -// .filter(|pli| pli.header.borrow().item_type == PlaylistItemType::SeriesInfo) -// .map(|pli| Rc::clone(&pli.header.borrow().uuid)).collect(); -// let mut series_playlist = download::get_xtream_playlist_series(provider_fpl, to_process_uuids, errors, resolve_series_delay).await; -// // original content saved into original list -// for plg in &series_playlist { -// provider_fpl.update_playlist(plg); -// } -// // run processing pipe over new items -// for f in pipe { -// let r = f(&mut series_playlist, target); -// if let Some(v) = r { -// series_playlist = v; -// } -// } -// // assign new items to the new playlist -// for plg in &series_playlist { -// processed_fpl.update_playlist(plg); -// } -// } -// } \ No newline at end of file diff --git a/src/repository/kodi_repository.rs b/src/repository/kodi_repository.rs index 95e569525..df09a860f 100644 --- a/src/repository/kodi_repository.rs +++ b/src/repository/kodi_repository.rs @@ -1,6 +1,6 @@ use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{Config, ConfigTarget}; -use crate::model::playlist::{PlaylistGroup, PlaylistItem, PlaylistItemType}; +use crate::model::playlist::{FieldAccessor, PlaylistGroup, PlaylistItem, PlaylistItemType}; use crate::model::xtream::XtreamSeriesEpisode; use crate::repository::bplustree::BPlusTree; use crate::repository::storage::get_input_storage_path; @@ -67,35 +67,44 @@ fn kodi_style_rename_episode(name: &str, style: &KodiStyle, series_episode: Opti extract_season_or_episode_match_with_default(name, &style.episode, series_episode) } -fn kodi_style_rename_year( - name: &str, +fn kodi_style_rename_year<'a>( + name: &'a str, style: &KodiStyle, - release_date: Option<&String>, -) -> (String, Option) { + release_date: Option<&'a String>, +) -> (&'a str, Option) { let mut years = Vec::new(); let cur_year = u32::try_from(chrono::Utc::now().year()).unwrap_or(0); - let new_name = style.year.replace_all(name, |caps: ®ex::Captures| { + let mut new_name = String::with_capacity(name.len()); + let mut last_index = 0; + + for caps in style.year.captures_iter(name) { if let Ok(year) = caps[0].parse::() { if (1900..=cur_year).contains(&year) { years.push(year); - return String::new(); + let match_start = caps.get(0).unwrap().start(); + let match_end = caps.get(0).unwrap().end(); + new_name.push_str(&name[last_index..match_start]); + last_index = match_end; } } - caps[0].to_string() - }).to_string(); + } + new_name.push_str(&name[last_index..]); let smallest_year = years.into_iter().min(); if smallest_year.is_none() { if let Some(rel_date) = release_date { - if let Some(year) = extract_match(rel_date, &style.year).1.and_then(|y| y.parse::().ok()) { - return (name.to_string(), Some(year)); + if let Some(year) = extract_match(rel_date, &style.year) + .1 + .and_then(|y| y.parse::().ok()) + { + return (name, Some(year)); } } } - (new_name, smallest_year) + (Box::leak(new_name.into_boxed_str()), smallest_year) } fn trim_string_after_pos(input: &str, start_pos: usize) -> Option { @@ -113,9 +122,9 @@ fn trim_whitespace(pattern: &Regex, input: &str) -> String { async fn kodi_style_rename(cfg: &Config, strm_item_info: &StrmItemInfo, style: &KodiStyle, input_tmdb_indexes: &mut InputTmdbIndexMap, underscore_whitespace: bool) -> (PathBuf, String) { let separator = if underscore_whitespace { "_" } else { " " }; - let (name_1, year) = kodi_style_rename_year(&strm_item_info.title, style, strm_item_info.release_date.as_ref()); - let mut series_year = year; - let (name_2, season) = kodi_style_rename_season(&name_1, style, strm_item_info.season.as_ref()); + let (name_1, year_from_title) = kodi_style_rename_year(&strm_item_info.title, style, strm_item_info.release_date.as_ref()); + let mut series_year = year_from_title; + let (name_2, season) = kodi_style_rename_season(name_1, style, strm_item_info.season.as_ref()); let (name_3, episode) = kodi_style_rename_episode(&name_2, style, strm_item_info.episode.as_ref()); let name_4 = trim_whitespace(&style.whitespace, &style.alphanumeric.replace_all(&name_3, "")); let title = &strm_item_info.series_name.as_ref() @@ -144,7 +153,7 @@ async fn kodi_style_rename(cfg: &Config, strm_item_info: &StrmItemInfo, style: & series_year = year; } } - folder_name + trim_whitespace(&style.whitespace, &style.alphanumeric.replace_all(folder_name, "")) } else { name_4 }; let sanitized_dir_name = sanitize_for_filename(&dir_name, underscore_whitespace); @@ -273,7 +282,10 @@ fn extract_item_info(pli: &PlaylistItem) -> StrmItemInfo { // TODO reverse proxy url let url = Rc::clone(&header.url); let (series_name, release_date, season, episode) = if header.item_type == PlaylistItemType::Series { - let series_name = header.get_additional_property_as_str("series_name"); + let series_name = match header.get_field("name") { + Some(name) if !name.is_empty() => Some(name.to_string()), + _ => header.get_additional_property_as_str("series_name"), + }; let release_date = header.get_additional_property_as_str("release_date"); let season = header.get_additional_property_as_str("season"); let episode = header.get_additional_property_as_str("episode"); diff --git a/test/rest-api.http b/test/rest-api.http index f9949eb66..5af3c18d6 100644 --- a/test/rest-api.http +++ b/test/rest-api.http @@ -30,6 +30,13 @@ username=xt&password=xt GET {{local}}/m3u?username=xtream&password=xtream.secret +### m3u playlist +GET {{local}}/m3u?username=xt&password=xt + +### m3u playlist +GET {{local}}/m3u?username=xt2&password=xt2 + + ### xtream login GET {{local}}/player_api.php?username=xt&password=xt @@ -57,9 +64,6 @@ GET {{local}}/player_api.php?username=xt&password=xt&action=get_series ### xtream series for category GET {{local}}/player_api.php?username=xt&password=xt&action=get_series&category_id=56 -### xtream series for category -GET {{local}}/player_api.php?username=xt&password=xt&action=get_series&category_id=56 - ### xtream series info GET {{local}}/player_api.php?username=xt&password=xt&action=get_series_info&series_id=13923