diff --git a/backend/src/library/classifier.rs b/backend/src/library/classifier.rs index 37808eae8..cfc65a15d 100644 --- a/backend/src/library/classifier.rs +++ b/backend/src/library/classifier.rs @@ -136,16 +136,28 @@ impl MediaClassifier { /// Extracts show name from file path for series /// Removes season/episode patterns and cleans up the name - pub fn extract_show_name(file: &ScannedMediaFile) -> String { + pub fn extract_show_name(file: &ScannedMediaFile) -> (Option>, String) { let file_name = Path::new(&file.file_name) .file_stem() .and_then(|s| s.to_str()) .unwrap_or(&file.file_name); + let moviedb_ids: Vec = CONSTANTS.re_classifier_moviedb_id.captures_iter(file_name) + .filter_map(|caps| { + let id = caps[2].parse::().ok()?; + match &caps[1].to_lowercase()[..] { + "tmdb" => Some(MovieDbId::Tmdb(id)), + "tvdb" => Some(MovieDbId::Tvdb(id)), + _ => None, + } + }) + .collect(); + + // Remove common series patterns let cleaned = CONSTANTS.re_classifier_cleanup.replace(file_name, "").trim().to_string(); // Clean up remaining special characters - clear_filename(&cleaned) + ((!moviedb_ids.is_empty()).then_some(moviedb_ids), clear_filename(&cleaned)) } /// Extracts movie title from file path diff --git a/backend/src/library/metadata.rs b/backend/src/library/metadata.rs index 82c1f7f52..a9b2a6b78 100644 --- a/backend/src/library/metadata.rs +++ b/backend/src/library/metadata.rs @@ -4,24 +4,32 @@ use uuid::Uuid; /// Source of metadata information #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum MetadataSource { - /// Metadata from Kodi NFO file #[default] + /// Metadata from TMDB API + Tmdb, + /// Metadata from Kodi NFO file KodiNfo, /// Metadata from Jellyfin/Emby metadata files JellyfinEmby, /// Metadata from Plex metadata files Plex, - /// Metadata from TMDB API - Tmdb, /// Metadata parsed from filename FilenameParsed, /// Manually entered metadata Manual, } +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct ExternalVideoMetadata { + pub name: String, //"Official Trailer", + pub key: String, + pub site: String, // "YouTube", + pub video_type: String, // "Trailer", "Teaser" + pub official: bool, +} + /// Movie metadata #[derive(Default, Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub struct MovieMetadata { pub title: String, // Original title (if different from title) @@ -64,11 +72,11 @@ pub struct MovieMetadata { pub fanart: Option, pub source: MetadataSource, pub last_updated: i64, + pub videos: Option>, } /// Series/TV show metadata #[derive(Default, Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub struct SeriesMetadata { pub title: String, // Original title (if different) @@ -133,7 +141,6 @@ pub struct EpisodeMetadata { /// Actor information #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub struct Actor { pub name: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -144,7 +151,6 @@ pub struct Actor { /// Complete video metadata (either movie or series) #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] #[serde(tag = "type")] pub enum MediaMetadata { #[serde(rename = "movie")] @@ -214,7 +220,6 @@ impl MediaMetadata { /// Metadata cache entry that links a file to its metadata #[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub struct MetadataCacheEntry { pub uuid: String, pub file_path: String, diff --git a/backend/src/library/metadata_resolver.rs b/backend/src/library/metadata_resolver.rs index 6e2963927..36cb75c79 100644 --- a/backend/src/library/metadata_resolver.rs +++ b/backend/src/library/metadata_resolver.rs @@ -5,7 +5,7 @@ use crate::library::metadata::{MediaMetadata, MetadataSource, MovieMetadata, Ser use crate::library::nfo_reader::NfoReader; use crate::library::scanner::ScannedMediaFile; use crate::library::tmdb_client::TmdbClient; -use crate::library::{MovieDbId, TMDB_API_KEY}; +use crate::library::{MetadataStorage, MovieDbId, TMDB_API_KEY}; use crate::model::LibraryConfig; /// Metadata resolver that tries multiple sources to get video metadata @@ -17,10 +17,10 @@ pub struct MetadataResolver { impl MetadataResolver { /// Creates a new metadata resolver from configuration - pub fn from_config(config: &LibraryConfig, client: reqwest::Client) -> Self { + pub fn from_config(config: &LibraryConfig, client: reqwest::Client, storage: MetadataStorage) -> Self { let tmdb_client = if config.metadata.tmdb.enabled { let api_key = config.metadata.tmdb.api_key.as_ref().map_or_else(|| TMDB_API_KEY.to_string(), ToString::to_string); - Some(TmdbClient::new(api_key, config.metadata.tmdb.rate_limit_ms, client)) + Some(TmdbClient::new(api_key, config.metadata.tmdb.rate_limit_ms, client, storage)) } else { None }; @@ -36,24 +36,30 @@ impl MetadataResolver { pub async fn resolve(&self, file: &ScannedMediaFile) -> Option { debug!("Resolving metadata for: {}", file.file_path); - // Step 1: Try to read existing NFO file - if let Some(metadata) = NfoReader::read_metadata(&file.path).await { - info!("Found NFO metadata for: {}", file.file_path); - return Some(metadata); - } - - // Step 2: Classify the file + // Step 1: Classify the file let classification = self.classifier.classify(file); debug!("Classified {} as: {classification:?}", file.file_path); - // Step 3: Try TMDB if enabled + // Step 2: Try TMDB if enabled if let Some(ref tmdb) = self.tmdb_client { - if let Some(metadata) = self.resolve_from_tmdb(file, &classification, tmdb).await { - info!("Found TMDB metadata for: {}", file.file_path); - return Some(metadata); + match self.resolve_from_tmdb(file, &classification, tmdb).await { + Ok(Some(metadata)) => { + info!("Found TMDB metadata for: {}", file.file_path); + return Some(metadata); + } + Ok(None) => { + return None; + } + Err(_err) => {}, } } + // Step 3: Try to read existing NFO file + if let Some(metadata) = NfoReader::read_metadata(&file.path).await { + info!("Found NFO metadata for: {}", file.file_path); + return Some(metadata); + } + // Step 4: Fallback to filename parsing if self.fallback_to_filename { info!("Using filename-based metadata for: {}", file.file_path); @@ -70,7 +76,7 @@ impl MetadataResolver { file: &ScannedMediaFile, classification: &MediaClassification, tmdb: &TmdbClient, - ) -> Option { + ) -> Result, String> { match classification { MediaClassification::Movie => { let (moviedb_ids, title, year) = MediaClassifier::extract_movie_search_info(file); @@ -78,7 +84,8 @@ impl MetadataResolver { tmdb.search_movie(tmdb_id, &title, year).await } MediaClassification::Series { .. } => { - let show_name = MediaClassifier::extract_show_name(file); + let (seriesdb_id, show_name) = MediaClassifier::extract_show_name(file); + let tmdb_id = MovieDbId::get_tmdb_id(seriesdb_id.as_ref()); debug!("Searching TMDB for series: {show_name}"); // Try to extract year from parent directory if available let year = file.path.parent() @@ -95,7 +102,7 @@ impl MetadataResolver { .filter(|&y| (1900..=2100).contains(&y)) }) }); - tmdb.search_series(&show_name, year).await + tmdb.search_series(tmdb_id, &show_name, year).await } } } @@ -121,7 +128,7 @@ impl MetadataResolver { }) } MediaClassification::Series { season: _, episode: _ } => { - let show_name = MediaClassifier::extract_show_name(file); + let (_moviedb_id, show_name) = MediaClassifier::extract_show_name(file); MediaMetadata::Series(SeriesMetadata { title: show_name, source: MetadataSource::FilenameParsed, @@ -196,7 +203,7 @@ mod tests { .timeout(Duration::from_secs(30)) .build() .unwrap_or_else(|_| reqwest::Client::new()); - let resolver = MetadataResolver::from_config(&config, client); + let resolver = MetadataResolver::from_config(&config, client, MetadataStorage::new(PathBuf::from("/tmp"))); let file = create_test_file("The.Matrix.1999.1080p.mkv"); let metadata = resolver.resolve(&file).await; @@ -220,7 +227,7 @@ mod tests { .unwrap_or_else(|_| reqwest::Client::new()); let mut config = create_test_config(false); config.metadata.fallback_to_filename = false; - let resolver = MetadataResolver::from_config(&config, client); + let resolver = MetadataResolver::from_config(&config, client, MetadataStorage::new(PathBuf::from("/tmp"))); let file = create_test_file("Unknown.Movie.mkv"); let metadata = resolver.resolve(&file).await; diff --git a/backend/src/library/metadata_storage.rs b/backend/src/library/metadata_storage.rs index a88069181..a3d05725e 100644 --- a/backend/src/library/metadata_storage.rs +++ b/backend/src/library/metadata_storage.rs @@ -6,19 +6,20 @@ use std::path::PathBuf; use tokio::fs; use tokio::io::AsyncWriteExt; -/// Metadata storage for local VOD files -/// Stores metadata as JSON files with UUID-based filenames +// Metadata storage for local VOD files +// Stores metadata as JSON files with UUID-based filenames +#[derive(Clone)] pub struct MetadataStorage { storage_dir: PathBuf, } impl MetadataStorage { - /// Creates a new metadata storage instance + // Creates a new metadata storage instance pub fn new(storage_dir: PathBuf) -> Self { Self { storage_dir } } - /// Initializes the storage directory + // Initializes the storage directory pub async fn initialize(&self) -> std::io::Result<()> { if !fs::try_exists(&self.storage_dir).await.unwrap_or(false) { info!("Creating metadata storage directory: {}", self.storage_dir.display()); @@ -27,7 +28,7 @@ impl MetadataStorage { Ok(()) } - /// Stores metadata for a video file + // Stores metadata for a video file pub async fn store(&self, entry: &MetadataCacheEntry) -> std::io::Result<()> { let file_path = self.get_metadata_file_path(&entry.uuid); @@ -43,7 +44,7 @@ impl MetadataStorage { Ok(()) } - /// Loads metadata for a specific UUID + // Loads metadata for a specific UUID pub async fn load_by_uuid(&self, uuid: &str) -> Option { let file_path = self.get_metadata_file_path(uuid); @@ -66,7 +67,7 @@ impl MetadataStorage { } } - /// Loads all metadata entries from storage + // Loads all metadata entries from storage pub async fn load_all(&self) -> Vec { let mut entries = Vec::new(); @@ -96,7 +97,7 @@ impl MetadataStorage { entries } - /// Deletes metadata for a specific UUID + // Deletes metadata for a specific UUID pub async fn delete_by_uuid(&self, uuid: &str) -> std::io::Result<()> { let file_path = self.get_metadata_file_path(uuid); @@ -108,7 +109,7 @@ impl MetadataStorage { Ok(()) } - /// Cleans up metadata for files that no longer exist + // Cleans up metadata for files that no longer exist pub async fn cleanup_orphaned(&self) -> std::io::Result { let entries = self.load_all().await; let mut deleted_count = 0; @@ -131,7 +132,7 @@ impl MetadataStorage { Ok(deleted_count) } - /// Builds a map of file paths to UUIDs for quick lookups + // Builds a map of file paths to UUIDs for quick lookups pub async fn build_path_index(&self) -> HashMap { let entries = self.load_all().await; entries @@ -141,27 +142,53 @@ impl MetadataStorage { } - /// Gets the metadata file path for a UUID + // Gets the metadata file path for a UUID fn get_metadata_file_path(&self, uuid: &str) -> PathBuf { self.storage_dir.join(format!("{uuid}.json")) } - /// Writes an NFO file for the given metadata + // write raw tmdb movie info + pub async fn store_tmdb_movie_info(&self, movie_id: u32, content: &[u8]) -> std::io::Result { + let file_path = self.get_metadata_file_path(&format!("tmdb_movie_{movie_id}")); + debug!("Storing raw tmdb movie metadata for {}", file_path.display()); + self.store_file(content, file_path).await + } + + // write raw tmdb series info + pub async fn store_tmdb_series_info(&self, series_id: u32, content: &[u8]) -> std::io::Result { + let file_path = self.get_metadata_file_path(&format!("tmdb_series_{series_id}")); + debug!("Storing raw tmdb series metadata for {}", file_path.display()); + self.store_file(content, file_path).await + } + + // write raw tmdb series season info + pub async fn store_tmdb_series_info_season(&self, series_id: u32, season: u32, content: &[u8]) -> std::io::Result { + let file_path = self.get_metadata_file_path(&format!("tmdb_series_{series_id}_{season}")); + debug!("Storing raw tmdb series season metadata for {}", file_path.display()); + self.store_file(content, file_path).await + } + + async fn store_file(&self, content: &[u8], file_path: PathBuf) -> std::io::Result { + let mut file = fs::File::create(&file_path).await?; + file.write_all(content).await?; + file.flush().await?; + Ok(file_path) + } + + // Writes an NFO file for the given metadata pub async fn write_nfo(&self, entry: &MetadataCacheEntry) -> std::io::Result<()> { let nfo_content = Self::generate_nfo_content(&entry.metadata); let nfo_path = PathBuf::from(entry.file_path.clone()).with_extension("nfo"); if !fs::try_exists(&nfo_path).await.unwrap_or(false) { debug!("Writing NFO file: {}", nfo_path.display()); - let mut file = fs::File::create(&nfo_path).await?; - file.write_all(nfo_content.as_bytes()).await?; - file.flush().await?; + self.store_file(nfo_content.as_bytes(), nfo_path).await?; } Ok(()) } - /// Generates NFO XML content from metadata + // Generates NFO XML content from metadata fn generate_nfo_content(metadata: &MediaMetadata) -> String { match metadata { MediaMetadata::Movie(movie) => { @@ -259,7 +286,7 @@ impl MetadataStorage { } } - /// Escapes XML special characters + // Escapes XML special characters fn xml_escape(s: &str) -> String { s.replace('&', "&") .replace('<', "<") diff --git a/backend/src/library/mod.rs b/backend/src/library/mod.rs index 8ed9d6de4..5f8775af9 100644 --- a/backend/src/library/mod.rs +++ b/backend/src/library/mod.rs @@ -7,6 +7,7 @@ mod metadata_storage; mod metadata_resolver; mod processor; mod metadata_iterator; +mod tmdb; pub use scanner::*; pub use classifier::*; diff --git a/backend/src/library/processor.rs b/backend/src/library/processor.rs index f27d00963..895d3541a 100644 --- a/backend/src/library/processor.rs +++ b/backend/src/library/processor.rs @@ -27,8 +27,8 @@ impl LibraryProcessor { pub fn new(config: LibraryConfig, client: reqwest::Client) -> Self { let storage_path = std::path::PathBuf::from(&config.metadata.path); let scanner = LibraryScanner::new(config.clone()); - let resolver = MetadataResolver::from_config(&config, client); let storage = MetadataStorage::new(storage_path); + let resolver = MetadataResolver::from_config(&config, client, storage.clone()); Self { config, @@ -157,7 +157,8 @@ impl LibraryProcessor { /// Writes metadata files (JSON, NFO) based on configuration async fn write_metadata_files(&self, entry: &MetadataCacheEntry) -> Result<(), std::io::Error> { // JSON is always written by storage.store() - // TODO should we remove json from formats ? + + // TODO enrich nfo with al information, we are currently storing a subset, and rebuilding json from nfo ends in information loss! // Write NFO if enabled if self.config.metadata.formats.contains(&LibraryMetadataFormat::Nfo) { if let Err(e) = self.storage.write_nfo(entry).await { diff --git a/backend/src/library/tmdb.rs b/backend/src/library/tmdb.rs new file mode 100644 index 000000000..c278e953d --- /dev/null +++ b/backend/src/library/tmdb.rs @@ -0,0 +1,393 @@ +use crate::library::metadata::{Actor, MetadataSource, MovieMetadata, SeriesMetadata}; +use crate::library::{ExternalVideoMetadata}; +use serde::Deserialize; + +const TMDB_IMAGE_BASE_URL: &str = "https://image.tmdb.org/t/p/w500"; + +// Helper function: Vec -> Option> +fn some_if_nonempty(v: Vec) -> Option> { + if v.is_empty() { None } else { Some(v) } +} + +// helper function: Crew-Filter +fn crew_names(credits: Option<&TmdbCredits>, jobs: &[&str]) -> Option> { + credits + .and_then(|c| c.crew.as_ref()) + .map(|crew| { + crew.iter() + .filter(|crew| { + crew.job + .as_ref() + .is_some_and(|j| jobs.contains(&j.as_str())) + }) + .map(|crew| crew.name.clone()) + .collect::>() + }) + .and_then(some_if_nonempty) +} + +// TMDB API response structures +#[derive(Debug, Deserialize)] +pub struct TmdbSearchResponse { + pub(crate) results: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbMovieSearchResult { + pub(crate) id: u32, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbTvSearchResponse { + pub(crate) results: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbTvSearchResult { + pub(crate) id: u32, +} + + +#[derive(Debug, Deserialize)] +pub struct TmdbExternalIds { + imdb_id: Option, + tvdb_id: Option, + // wikidata_id: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbVideo { + name: String, //"Official Trailer", + key: String, + site: String, // "YouTube", + size: u32, + #[serde(rename = "type")] + video_type: String, // "Trailer", "Teaser" + official: bool, + + // "iso_639_1": "en", + // "iso_3166_1": "US", + // "published_at": "2019-05-31T02:00:01.000Z", + // "id": "5cf20e7dc3a368697a2032d1" +} + +impl TmdbVideo { + pub fn to_meta_data(&self) -> ExternalVideoMetadata { + ExternalVideoMetadata { + name: self.name.clone(), + key: self.key.clone(), + site: self.site.to_lowercase(), + video_type: self.video_type.clone(), + official: self.official, + } + } +} + +#[derive(Debug, Deserialize)] +pub struct TmdbVideos { + results: Option>, +} + +impl TmdbVideos { + pub fn get_youtube(&self) -> Option> { + self.results.as_ref().map(|r| { + r.iter() + .filter(|v| v.site.eq_ignore_ascii_case("youtube")) + .cloned() + .collect() + }) + } +} + +#[derive(Debug, Deserialize)] +pub struct TmdbMovieDetails { + id: u32, + title: String, + original_title: String, + #[serde(default)] + overview: String, + tagline: Option, + #[serde(default)] + release_date: String, + #[serde(default)] + runtime: u32, + #[serde(default)] + vote_average: f64, + imdb_id: Option, + poster_path: Option, + backdrop_path: Option, + genres: Option>, + production_companies: Option>, + credits: Option, + external_ids: Option, + videos: Option, +} + +impl TmdbMovieDetails { + pub(crate) fn to_meta_data(&self) -> MovieMetadata { + MovieMetadata { + title: self.title.clone(), + original_title: Some(self.original_title.clone()), + year: self.release_date.split('-').next().and_then(|y| y.parse().ok()), + plot: Some(self.overview.clone()), + tagline: self.tagline.clone(), + runtime: Some(self.runtime), + mpaa: None, + tmdb_id: Some(self.id), + imdb_id: self.imdb_id.clone().or_else(|| self.external_ids.as_ref().and_then(|ei| ei.imdb_id.clone())), + tvdb_id: self.external_ids.as_ref().and_then(|ei| ei.tvdb_id.clone()), + rating: Some(self.vote_average), + genres: self.genres.as_ref().map(|list| list.iter().map(|g| g.name.clone()).collect()).and_then(some_if_nonempty), + directors: crew_names(self.credits.as_ref(), &["Director"]), + writers: crew_names(self.credits.as_ref(), &["Writer", "Screenplay"]), + actors: self.credits + .as_ref() + .and_then(|c| c.cast.as_ref()) + .map(|cast| { + cast.iter() + .take(10) + .map(|actor| Actor { + name: actor.name.clone(), + role: Some(actor.character.clone()), + thumb: actor.profile_path + .as_ref() + .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), + }) + .collect::>() + }) + .and_then(some_if_nonempty), + studios: self.production_companies.as_ref().map(|list| list.iter().map(|n| n.name.clone()).collect()).and_then(some_if_nonempty), + poster: as Clone>::clone(&self.poster_path).map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), + fanart: self.backdrop_path.clone().map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), + source: MetadataSource::Tmdb, + last_updated: chrono::Utc::now().timestamp(), + videos: self.videos + .as_ref() + .and_then(|v| v.get_youtube()) + .map(|list| { + list.iter() + .map(|v| v.to_meta_data()) + .collect::>() + }).filter(|v| !v.is_empty()), + } + } +} + +#[derive(Debug, Deserialize)] +pub struct TmdbSeriesInfoEpisodeDetails { + pub air_date: Option, + pub episode_number: u32, + pub episode_type: String, + + pub id: u32, + pub name: String, + pub overview: String, + + pub production_code: String, + pub runtime: Option, + + pub season_number: u32, + pub show_id: u32, + + pub still_path: Option, + + pub vote_average: f32, + pub vote_count: u32, + + pub crew: Vec, + pub guest_stars: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbSeriesInfoSeasonDetails { + #[serde(rename = "_id")] + pub internal_id: String, + + pub air_date: Option, + pub name: String, + pub overview: String, + + pub id: u32, + pub season_number: u32, + pub vote_average: f32, + + pub poster_path: Option, + + pub episodes: Vec, + pub networks: Vec, + + pub credits: Option, +} + + +#[derive(Debug, Deserialize)] +pub struct TmdbSeriesInfoDetails { + id: u32, + name: String, + original_name: String, + #[serde(default)] + overview: String, + #[serde(default)] + first_air_date: String, + last_air_date: Option, + #[serde(default)] + number_of_episodes: u32, + #[serde(default)] + pub(crate) number_of_seasons: u32, + #[serde(default)] + vote_average: f64, + #[serde(default)] + popularity: f64, + poster_path: Option, + backdrop_path: Option, + #[serde(default)] + status: String, + genres: Option>, + networks: Option>, + credits: Option, + #[serde(default)] + adult: bool, + homepage: Option, + #[serde(default)] + in_production: bool, + languages: Option>, + pub(crate) seasons: Option>, + external_ids: Option, + videos: Option, + pub(crate) episodes: Option>, +} + +impl TmdbSeriesInfoDetails { + pub(crate) fn to_meta_data(&self) -> SeriesMetadata { + SeriesMetadata { + title: self.name.clone(), + original_title: Some(self.original_name.clone()), + year: self + .first_air_date + .split('-') + .next() + .and_then(|y| y.parse().ok()), + plot: Some(self.overview.clone()), + mpaa: None, + imdb_id: None, // TMDB TV doesn't always provide IMDB ID + tmdb_id: Some(self.id), + tvdb_id: None, // TMDB doesn't provide TVDB ID directly + rating: Some(self.vote_average), + genres: self.genres.as_ref().and_then(|list| { + let result: Vec = list.iter().map(|g| g.name.clone()).collect(); + if result.is_empty() { + None + } else { + Some(result) + } + }), + actors: self.credits + .as_ref() + .and_then(|c| c.cast.as_ref()) + .map(|cast| { + cast.iter() + .take(10) + .map(|actor| Actor { + name: actor.name.clone(), + role: Some(actor.character.clone()), + thumb: actor.profile_path + .as_ref() + .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), + }) + .collect::>() + }) + .filter(|v| !v.is_empty()), + studios: self.networks.as_ref().and_then(|list| { + let result: Vec = list.iter().map(|n| n.name.clone()).collect(); + if result.is_empty() { + None + } else { + Some(result) + } + }), + poster: self + .poster_path.clone() + .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), + fanart: self + .backdrop_path.clone() + .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), + status: Some(self.status.clone()), + episodes: None, // Episodes would need separate API calls + source: MetadataSource::Tmdb, + last_updated: chrono::Utc::now().timestamp(), + } + } +} + +#[derive(Debug, Deserialize)] +pub struct TmdbGenre { + name: String, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbCompany { + name: String, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbNetwork { + name: String, + // id: u32, + // logo_path: Option, + // origin_country: String, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbCredits { + cast: Option>, + crew: Option>, +} + +#[derive(Debug, Deserialize)] +struct TmdbCast { + name: String, + #[serde(default)] + character: String, + profile_path: Option, + + // adult: bool, + // gender: Option, + // id: u32, + // original_name: String, + // known_for_department: String, + // popularity: f32, + // credit_id: String, + // order: u32, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbCrew { + pub name: String, + pub job: Option, + + // adult: Option, + // gender: Option, + // id: u32, + // original_name: String, + // known_for_department: String, + // popularity: f32, + // profile_path: Option, + // credit_id: String, + // department: Option, +} + +#[derive(Debug, Deserialize)] +pub struct TmdbSeason { + id: u32, + air_date: Option, + #[serde(default)] + episode_count: u32, + #[serde(default)] + name: String, + overview: Option, + poster_path: Option, + #[serde(default)] + season_number: u32, + #[serde(default)] + vote_average: f64, +} diff --git a/backend/src/library/tmdb_client.rs b/backend/src/library/tmdb_client.rs index 9246cce83..d2b9a0c9f 100644 --- a/backend/src/library/tmdb_client.rs +++ b/backend/src/library/tmdb_client.rs @@ -1,51 +1,43 @@ -use crate::library::metadata::{Actor, MediaMetadata, MetadataSource, MovieMetadata, SeriesMetadata}; +use crate::library::metadata::{MediaMetadata}; +use crate::library::{MetadataStorage}; use log::{debug, error, warn}; -use serde::Deserialize; +use std::collections::{HashSet}; use tokio::time::{sleep, Duration}; use url::Url; +use crate::library::tmdb::{TmdbMovieDetails, TmdbSearchResponse, TmdbSeriesInfoDetails, TmdbSeriesInfoSeasonDetails, TmdbTvSearchResponse}; pub const TMDB_API_KEY: &str = "4219e299c89411838049ab0dab19ebd5"; // TODO make this configurable in Library tmdb config const TMDB_API_BASE_URL: &str = "https://api.themoviedb.org/3"; -const TMDB_IMAGE_BASE_URL: &str = "https://image.tmdb.org/t/p/w500"; - - -// Helper function: Vec -> Option> -fn some_if_nonempty(v: Vec) -> Option> { - if v.is_empty() { None } else { Some(v) } -} - -// helper function: Crew-Filter -fn crew_names(credits: Option<&TmdbCredits>, jobs: &[&str]) -> Option> { - credits.as_ref().map(|c| { - c.crew - .iter() - .filter(|crew| jobs.contains(&crew.job.as_str())) - .map(|crew| crew.name.clone()) - .collect::>() - }).and_then(some_if_nonempty) -} /// TMDB API client with rate limiting pub struct TmdbClient { api_key: String, client: reqwest::Client, rate_limit_ms: u64, + storage: MetadataStorage, + fetched_movie_metadata: tokio::sync::RwLock>, + fetched_series_metadata: tokio::sync::RwLock>, + fetched_series_key: tokio::sync::RwLock>, } impl TmdbClient { /// Creates a new TMDB client - pub fn new(api_key: String, rate_limit_ms: u64, client: reqwest::Client) -> Self { + pub fn new(api_key: String, rate_limit_ms: u64, client: reqwest::Client, storage: MetadataStorage) -> Self { Self { api_key, client, rate_limit_ms, + storage, + fetched_movie_metadata: tokio::sync::RwLock::new(HashSet::new()), + fetched_series_metadata: tokio::sync::RwLock::new(HashSet::new()), + fetched_series_key: tokio::sync::RwLock::new(HashSet::new()), } } /// Searches for a movie by title and optional year - pub async fn search_movie(&self, tmdb_id: Option, title: &str, year: Option) -> Option { + pub async fn search_movie(&self, tmdb_id: Option, title: &str, year: Option) -> Result, String> { debug!("TMDB search movie: {title}"); if let Some(movie_id) = tmdb_id { @@ -59,7 +51,7 @@ impl TmdbClient { Ok(url) => url, Err(err) => { error!("Failed to parse URL for tmdb movie search: {err}"); - return None; + return Err(format!("Failed to parse URL for tmdb movie search: {err}")); } }; url.query_pairs_mut().append_pair("api_key", &self.api_key); @@ -78,81 +70,71 @@ impl TmdbClient { self.fetch_movie_details(movie.id).await } else { debug!("No TMDB results for movie: {title}"); - None + Ok(None) } } Err(e) => { error!("Failed to parse TMDB search response: {e}"); - None + Err(format!("Failed to parse TMDB search response: {e}")) } } } else { warn!("TMDB API error: {}", response.status()); - None + Err(format!("TMDB API error: {}", response.status())) } } Err(e) => { error!("TMDB API request failed: {e}"); - None + Err(format!("TMDB API request failed: {e}")) } } } /// Fetches detailed movie information - async fn fetch_movie_details(&self, movie_id: u32) -> Option { - - sleep(Duration::from_millis(self.rate_limit_ms)).await; - - let url = format!("{TMDB_API_BASE_URL}/movie/{movie_id}?api_key={}&append_to_response=credits", self.api_key); - - let response = self.client.get(&url).send().await.ok()?; - if !response.status().is_success() { - warn!("TMDB API error fetching movie details: {}", response.status()); - return None; + async fn fetch_movie_details(&self, movie_id: u32) -> Result, String> { + if self.fetched_movie_metadata.read().await.contains(&movie_id) { + return Ok(None); } - let details: TmdbMovieDetails = response.json().await.ok()?; + sleep(Duration::from_millis(self.rate_limit_ms)).await; + let url = format!("{TMDB_API_BASE_URL}/movie/{movie_id}?api_key={}&append_to_response=credits,videos,external_ids", self.api_key); + let response = self.client.get(&url).send().await.map_err(|err| err.to_string())?; + if !response.status().is_success() { + warn!("TMDB API error fetching movie details: {}", response.status()); + return Err(format!("TMDB API error fetching movie details: {}", response.status())); + } - Some(MediaMetadata::Movie(MovieMetadata { - title: details.title, - original_title: Some(details.original_title), - year: details.release_date.split('-').next().and_then(|y| y.parse().ok()), - plot: Some(details.overview), - tagline: details.tagline, - runtime: Some(details.runtime), - mpaa: None, - imdb_id: details.imdb_id, - tmdb_id: Some(details.id), - tvdb_id: None, - rating: Some(details.vote_average), - genres: details.genres.as_ref().map(|list| list.iter().map(|g| g.name.clone()).collect()).and_then(some_if_nonempty), - directors: crew_names(details.credits.as_ref(), &["Director"]), - writers: crew_names(details.credits.as_ref(), &["Writer", "Screenplay"]), - actors: details.credits.as_ref().and_then(|c| { - some_if_nonempty(c.cast.iter().take(10).map(|actor| Actor { - name: actor.name.clone(), - role: Some(actor.character.clone()), - thumb: actor.profile_path.as_ref().map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), - }).collect::>()) - }), - studios: details.production_companies.as_ref().map(|list| list.iter().map(|n| n.name.clone()).collect()).and_then(some_if_nonempty), - poster: details.poster_path.map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), - fanart: details.backdrop_path.map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), - source: MetadataSource::Tmdb, - last_updated: chrono::Utc::now().timestamp(), - })) + let content_bytes = response.bytes().await.map_err(|err| err.to_string())?; + let _raw_data_path = self.storage.store_tmdb_movie_info(movie_id, &content_bytes).await.map_err(|err| err.to_string())?; + let details: TmdbMovieDetails = serde_json::from_slice(&content_bytes).map_err(|err| err.to_string())?; + self.fetched_movie_metadata.write().await.insert(movie_id); + + Ok(Some(MediaMetadata::Movie(details.to_meta_data()))) } /// Searches for a TV series by title and optional year - pub async fn search_series(&self, title: &str, year: Option) -> Option { + pub async fn search_series(&self, tmdb_id: Option, title: &str, year: Option) -> Result, String> { + let key = format!("{title}-{tmdb_id:?}-{year:?}"); + if self.fetched_series_key.read().await.contains(&key) { + return Ok(None); + } + + if let Some(series_id) = tmdb_id { + let result = self.fetch_series_details(series_id).await; + if let Ok(Some(_)) = result { + self.fetched_series_key.write().await.insert(key); + } + return result; + } + sleep(Duration::from_millis(self.rate_limit_ms)).await; let mut url = match Url::parse(&format!("{TMDB_API_BASE_URL}/search/tv")) { Ok(url) => url, Err(err) => { error!("Failed to parse URL for tmdb series search: {err}"); - return None; + return Err(format!("Failed to parse URL for tmdb series search: {err}")); } }; url.query_pairs_mut().append_pair("api_key", &self.api_key); @@ -163,7 +145,7 @@ impl TmdbClient { debug!("TMDB search series: {title}"); - match self.client.get(url).send().await { + let result = match self.client.get(url).send().await { Ok(response) => { if response.status().is_success() { match response.json::().await { @@ -172,212 +154,102 @@ impl TmdbClient { self.fetch_series_details(series.id).await } else { debug!("No TMDB results for series: {title}"); - None + Ok(None) } } Err(e) => { error!("Failed to parse TMDB TV search response: {e}"); - None + Err(format!("Failed to parse TMDB TV search response: {e}")) } } } else { warn!("TMDB API error: {}", response.status()); - None + Err(format!("TMDB API error: {}", response.status())) } } Err(e) => { error!("TMDB API request failed: {e}"); - None + Err(format!("TMDB API request failed: {e}")) } + }; + if let Ok(Some(_)) = result { + self.fetched_series_key.write().await.insert(key); } + result } /// Fetches detailed TV series information - async fn fetch_series_details(&self, series_id: u32) -> Option { + async fn fetch_series_details(&self, series_id: u32) -> Result, String> { + if self.fetched_series_metadata.read().await.contains(&series_id) { + return Ok(None); + } + sleep(Duration::from_millis(self.rate_limit_ms)).await; - let url = format!("{TMDB_API_BASE_URL}/tv/{series_id}?api_key={}&append_to_response=credits", self.api_key); + let url = format!("{TMDB_API_BASE_URL}/tv/{series_id}?api_key={}&append_to_response=credits,videos,external_ids", self.api_key); match self.client.get(&url).send().await { Ok(response) => { if response.status().is_success() { - match response.json::().await { - Ok(details) => Some(MediaMetadata::Series(SeriesMetadata { - title: details.name, - original_title: Some(details.original_name), - year: details - .first_air_date - .split('-') - .next() - .and_then(|y| y.parse().ok()), - plot: Some(details.overview), - mpaa: None, - imdb_id: None, // TMDB TV doesn't always provide IMDB ID - tmdb_id: Some(details.id), - tvdb_id: None, // TMDB doesn't provide TVDB ID directly - rating: Some(details.vote_average), - genres: details.genres.as_ref().and_then(|list| { - let result: Vec = list.iter().map(|g| g.name.clone()).collect(); - if result.is_empty() { - None - } else { - Some(result) - } - }), - actors: details - .credits - .as_ref() - .and_then(|c| { - let actors: Vec = c.cast - .iter() - .take(10) // Limit to top 10 actors - .map(|actor| Actor { - name: actor.name.clone(), - role: Some(actor.character.clone()), - thumb: actor - .profile_path - .as_ref() - .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), - }) - .collect(); + let content_bytes = response.bytes().await.map_err(|err| err.to_string())?; + let _raw_data_path = self.storage.store_tmdb_series_info(series_id, &content_bytes).await.map_err(|err| err.to_string())?; + self.fetched_series_metadata.write().await.insert(series_id); - if actors.is_empty() { - None - } else { - Some(actors) + match serde_json::from_slice::(&content_bytes) { + Ok(mut series_details) => { + let seasons = if series_details.number_of_seasons > 0 { + series_details.number_of_seasons + } else { + series_details.seasons.as_ref().map(|s| u32::try_from(s.len()).unwrap_or(0)).unwrap_or(0) + }; + if seasons > 0 { + let mut seasons_info = vec![]; + for season in 1..=seasons { + let url = format!("{TMDB_API_BASE_URL}/tv/{series_id}/season/{season}/?api_key={}&append_to_response=credits", self.api_key); + match self.client.get(&url).send().await { + Ok(response) => { + match response.bytes().await { + Ok(season_bytes) => { + let _ = self.storage.store_tmdb_series_info_season(series_id, season, &season_bytes).await; + match serde_json::from_slice::(&content_bytes) { + Ok(season_details) => { + seasons_info.push(season_details); + } + Err(err) => { + error!("Failed to parse series season details: tmdb-id: {series_id} season: {season} err: {err}"); + } + } + } + Err(err) => { + error!("Failed to fetch series season details: tmdb-id: {series_id} season: {season} err: {err}"); + } + } + } + Err(err) => error!("Failed to fetch series season details: tmdb-id: {series_id} season: {season} err: {err}"), } - }), - studios: details.networks.as_ref().and_then(|list| { - let result: Vec = list.iter().map(|n| n.name.clone()).collect(); - if result.is_empty() { - None - } else { - Some(result) } - }), - poster: details - .poster_path - .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), - fanart: details - .backdrop_path - .map(|p| format!("{TMDB_IMAGE_BASE_URL}{p}")), - status: Some(details.status), - episodes: None, // Episodes would need separate API calls - source: MetadataSource::Tmdb, - last_updated: chrono::Utc::now().timestamp(), - })), + + if !seasons_info.is_empty() { + series_details.episodes = Some(seasons_info); + } + } + + Ok(Some(MediaMetadata::Series(series_details.to_meta_data()))) + } Err(e) => { error!("Failed to parse TMDB series details: {e}"); - None + Err(format!("Failed to parse TMDB series details: {e}")) } } } else { warn!("TMDB API error fetching series details: {}", response.status()); - None + Err(format!("TMDB API error fetching series details: {}", response.status())) } } Err(e) => { error!("TMDB API request failed: {e}"); - None + Err(format!("TMDB API request failed: {e}")) } } } } - -// TMDB API response structures -#[derive(Debug, Deserialize)] -struct TmdbSearchResponse { - results: Vec, -} - -#[derive(Debug, Deserialize)] -struct TmdbMovieSearchResult { - id: u32, -} - -#[derive(Debug, Deserialize)] -struct TmdbTvSearchResponse { - results: Vec, -} - -#[derive(Debug, Deserialize)] -struct TmdbTvSearchResult { - id: u32, -} - -#[derive(Debug, Deserialize)] -struct TmdbMovieDetails { - id: u32, - title: String, - original_title: String, - #[serde(default)] - overview: String, - tagline: Option, - #[serde(default)] - release_date: String, - #[serde(default)] - runtime: u32, - #[serde(default)] - vote_average: f64, - imdb_id: Option, - poster_path: Option, - backdrop_path: Option, - genres: Option>, - production_companies: Option>, - credits: Option, -} - -#[derive(Debug, Deserialize)] -struct TmdbSeriesDetails { - id: u32, - name: String, - original_name: String, - #[serde(default)] - overview: String, - #[serde(default)] - first_air_date: String, - vote_average: f64, - poster_path: Option, - backdrop_path: Option, - status: String, - genres: Option>, - networks: Option>, - credits: Option, -} - -#[derive(Debug, Deserialize)] -struct TmdbGenre { - name: String, -} - -#[derive(Debug, Deserialize)] -struct TmdbCompany { - name: String, -} - -#[derive(Debug, Deserialize)] -struct TmdbNetwork { - name: String, -} - -#[derive(Debug, Deserialize)] -struct TmdbCredits { - #[serde(default)] - cast: Vec, - #[serde(default)] - crew: Vec, -} - -#[derive(Debug, Deserialize)] -struct TmdbCast { - name: String, - #[serde(default)] - character: String, - profile_path: Option, -} - -#[derive(Debug, Deserialize)] -struct TmdbCrew { - name: String, - job: String, -} diff --git a/backend/src/processing/processor/xtream_series.rs b/backend/src/processing/processor/xtream_series.rs index ba13f5ce1..09449ed8b 100644 --- a/backend/src/processing/processor/xtream_series.rs +++ b/backend/src/processing/processor/xtream_series.rs @@ -7,7 +7,9 @@ use crate::processing::processor::xtream::normalize_json_content; use crate::processing::processor::xtream::{create_resolve_episode_wal_files, create_resolve_info_wal_files, playlist_resolve_download_playlist_item, read_processed_info_ids, should_update_info}; use crate::processing::processor::{create_resolve_options_function_for_xtream_target, handle_error, handle_error_and_return}; use crate::repository::storage::get_input_storage_path; -use crate::repository::xtream_repository::{write_series_info_to_wal_file, xtream_get_info_file_paths, xtream_update_input_info_file, xtream_update_input_series_episodes_record_from_wal_file, xtream_update_input_series_record_from_wal_file}; +use crate::repository::xtream_repository::{write_series_info_to_wal_file, xtream_get_info_file_paths, + xtream_update_input_info_file, xtream_update_input_series_episodes_record_from_wal_file, + xtream_update_input_series_record_from_wal_file}; use crate::repository::IndexedDocumentReader; use crate::utils; use crate::utils::{bincode_serialize, IO_BUFFER_SIZE}; @@ -53,7 +55,6 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, fpl: &mut FetchedPlaylist<'_>, resolve_delay: u16) -> bool { // TODO read existing WAL File and import it to avoid duplicate requests - let mut processed_info_ids: HashMap = read_processed_series_info_ids(cfg, errors, fpl).await; let mut fetched_in_run: HashSet = HashSet::new(); // we can't write to the indexed-document directly because of the write lock and time-consuming operation. @@ -70,20 +71,21 @@ async fn playlist_resolve_series_info(cfg: &AppConfig, client: &reqwest::Client, let series_info_count = fpl.playlistgroups.iter() .filter(|&plg| plg.xtream_cluster == XtreamCluster::Series) .flat_map(|plg| &plg.channels) - .filter(|&pli| matches!(pli.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo)).count(); - + .filter(|&pli| pli.header.item_type == PlaylistItemType::SeriesInfo).count(); info!("Found {series_info_count} series info to resolve"); let mut last_log_time = Instant::now(); let mut processed_series_info_count = 0; let mut write_counter = 0usize; + // LocalSeriesInfo entries are not resolved! + for plg in &mut fpl.playlistgroups { if plg.xtream_cluster != XtreamCluster::Series { continue; } for pli in &mut plg.channels { - if !matches!(pli.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo) { + if pli.header.item_type != PlaylistItemType::SeriesInfo { continue; } let (should_update, provider_id, ts) = should_update_series_info(pli, &processed_info_ids); @@ -178,10 +180,11 @@ async fn process_series_info( { let mut group_series = vec![]; + // Resolve does not handle LocalSeriesInfo for pli in plg .channels .iter_mut() - .filter(|pli| matches!(pli.header.item_type, PlaylistItemType::SeriesInfo | PlaylistItemType::LocalSeriesInfo)) + .filter(|pli| pli.header.item_type == PlaylistItemType::SeriesInfo) { let Some(provider_id) = pli.header.get_provider_id() else { continue; }; let Ok(content) = info_reader.get(&provider_id) else { continue; }; diff --git a/backend/src/processing/processor/xtream_vod.rs b/backend/src/processing/processor/xtream_vod.rs index 2d0a3e4cb..b70774463 100644 --- a/backend/src/processing/processor/xtream_vod.rs +++ b/backend/src/processing/processor/xtream_vod.rs @@ -85,10 +85,12 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Clie let mut record_writer = utils::async_file_writer(&mut wal_record_file); let mut content_updated = false; + // LocalVideo entries are not resolved! + // TODO merge both filters to one let vod_info_count = fpl.playlistgroups.iter() .flat_map(|plg| &plg.channels) - .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Video).count(); + .filter(|&pli| pli.header.xtream_cluster == XtreamCluster::Video && pli.header.item_type == PlaylistItemType::Video).count(); info!("Found {vod_info_count} vod info to resolve"); let mut last_log_time = Instant::now(); @@ -97,7 +99,8 @@ pub async fn playlist_resolve_vod(app_config: &AppConfig, client: &reqwest::Clie for plg in &mut fpl.playlistgroups { for pli in &mut plg.channels { - if pli.header.xtream_cluster != XtreamCluster::Video { + // LocalVideo files are not resolved + if pli.header.xtream_cluster != XtreamCluster::Video && pli.header.item_type != PlaylistItemType::Video { continue; } let (should_update, provider_id, _ts) = should_update_vod_info(pli, &processed_info_ids);