package catalogseed import ( "bytes" "compress/gzip" "context" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "sort" "strconv" "strings" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" pgvector "github.com/pgvector/pgvector-go" "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/idgen" "github.com/Silo-Server/silo-server/internal/lang" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/recommendations" "github.com/Silo-Server/silo-server/internal/titleutil" ) var ( ErrUnsupportedBundleVersion = errors.New("unsupported catalog seed version") ErrInvalidConflictMode = errors.New("invalid catalog seed conflict mode") ErrInvalidBundle = errors.New("invalid catalog seed bundle") ) type UnmatchedRootsError struct { Roots []string } func (e *UnmatchedRootsError) Error() string { if len(e.Roots) == 0 { return "catalog seed import requires additional path rewrites" } return fmt.Sprintf("catalog seed import requires additional path rewrites: %s", strings.Join(e.Roots, ", ")) } type Service struct { pool *pgxpool.Pool personRepo *catalog.PersonRepository recsRepo *recommendations.Repo } func NewService(pool *pgxpool.Pool, personRepo *catalog.PersonRepository, recsRepo *recommendations.Repo) *Service { if personRepo == nil { personRepo = catalog.NewPersonRepository(pool) } if recsRepo == nil { recsRepo = recommendations.NewRepo(pool) } return &Service{pool: pool, personRepo: personRepo, recsRepo: recsRepo} } func (s *Service) Export(ctx context.Context, opts ExportOptions) ([]byte, error) { var raw bytes.Buffer if _, err := s.ExportToWriter(ctx, &raw, opts, nil); err != nil { return nil, err } return raw.Bytes(), nil } func (s *Service) Import(ctx context.Context, data []byte, opts ImportOptions) (*ImportResult, error) { return s.ImportWithProgress(ctx, data, opts, nil) } func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts ImportOptions, progress func(ImportProgress)) (*ImportResult, error) { if opts.ConflictMode == "" { opts.ConflictMode = ConflictModeSkipExisting } if opts.ConflictMode != ConflictModeSkipExisting && opts.ConflictMode != ConflictModeOverwrite { return nil, ErrInvalidConflictMode } reportProgress := func(message string, current, total int) { if progress == nil { return } progress(ImportProgress{ Message: message, Current: current, Total: total, }) } reportProgress("Reading catalog bundle", 0, 0) bundle, err := decodeBundle(data) if err != nil { return nil, err } if bundle.Manifest.FormatVersion != CurrentBundleVersion { return nil, ErrUnsupportedBundleVersion } for i := range bundle.Items { bundle.Items[i].OriginalLanguage = lang.Canonical(bundle.Items[i].OriginalLanguage) bundle.Items[i].Countries = lang.CanonicalCountries(bundle.Items[i].Countries) } totalWork := len(bundle.Libraries) + len(bundle.Items) + len(bundle.Embeddings) + len(bundle.Seasons) + len(bundle.Episodes) + len(bundle.Files) + len(bundle.LibraryLinks) + 1 if totalWork <= 0 { totalWork = 1 } currentWork := 0 reportProgress("Validating catalog bundle", currentWork, totalWork) rewrites := normalizeRewrites(opts.PathRewrites) libraries, folderPaths, unmatchedRoots, err := prepareLibraries(bundle.Libraries, rewrites) if err != nil { return nil, err } if len(unmatchedRoots) > 0 { sort.Strings(unmatchedRoots) return nil, &UnmatchedRootsError{Roots: unmatchedRoots} } if err := validateFileRoots(bundle.Files, folderPaths, rewrites); err != nil { return nil, err } reportProgress("Starting catalog import", currentWork, totalWork) tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) if err != nil { return nil, fmt.Errorf("beginning catalog seed import: %w", err) } defer func() { _ = tx.Rollback(ctx) }() emptyTarget, err := isEmptyCatalogImportTarget(ctx, tx) if err != nil { return nil, err } result := &ImportResult{} folderIDMap := make(map[int]int, len(libraries)) for _, library := range libraries { localID, created, matched, importErr := importLibrary(ctx, tx, library, opts.ConflictMode == ConflictModeOverwrite) if importErr != nil { return nil, importErr } folderIDMap[library.ExportedID] = localID if created { result.LibrariesCreated++ } if matched { result.LibrariesMatched++ } currentWork++ reportProgress("Importing libraries", currentWork, totalWork) } if emptyTarget { // Empty-target fast path: use COPY protocol for maximum throughput. itemRows := make([][]any, 0, len(bundle.Items)) for _, item := range bundle.Items { studios, networks, countries, keywords := itemRecordStringArrays(item) itemRows = append(itemRows, []any{ item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres, item.ContentRating, item.Runtime, item.Overview, item.Tagline, item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience, item.ImdbID, item.TmdbID, item.TvdbID, item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath, item.MetadataS3Path, item.MetadataEtag, item.SeasonCount, studios, networks, countries, keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate, item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status, item.CreatedAt, item.UpdatedAt, }) } if err := copyInsertBatches(ctx, tx, "media_items", []string{ "content_id", "type", "title", "sort_title", "original_title", "year", "genres", "content_rating", "runtime", "overview", "tagline", "rating_imdb", "rating_tmdb", "rating_rt_critic", "rating_rt_audience", "imdb_id", "tmdb_id", "tvdb_id", "poster_path", "poster_thumbhash", "backdrop_path", "backdrop_thumbhash", "logo_path", "metadata_s3_path", "metadata_etag", "season_count", "studios", "networks", "countries", "keywords", "original_language", "release_date", "first_air_date", "last_air_date", "matched_at", "last_refreshed", "refresh_failures", "locked_fields", "status", "created_at", "updated_at", }, itemRows, func(processed int) { currentWork += processed reportProgress("Importing catalog items", currentWork, totalWork) }); err != nil { return nil, err } result.ItemsCreated = len(bundle.Items) // Build lookup sets for FK filtering of child records. itemIDs := make(map[string]struct{}, len(bundle.Items)) itemStates := make(map[string]bool, len(bundle.Items)) for _, item := range bundle.Items { itemIDs[item.ContentID] = struct{}{} itemStates[item.ContentID] = true } reportProgress("Importing people", currentWork, totalWork) if err := s.replacePeople(ctx, tx, bundle.People, itemStates, ConflictModeOverwrite, result); err != nil { return nil, err } if err := s.importEmbeddings(ctx, tx, bundle.Embeddings, true, result, func(processed int) { currentWork += processed reportProgress("Importing embeddings", currentWork, totalWork) }); err != nil { return nil, err } currentWork++ reportProgress("Importing embeddings", currentWork, totalWork) seasonRows := make([][]any, 0, len(bundle.Seasons)) seasonIDs := make(map[string]struct{}, len(bundle.Seasons)) for _, season := range bundle.Seasons { if _, ok := itemIDs[season.SeriesID]; !ok { continue // skip orphan season } seasonIDs[season.ContentID] = struct{}{} seasonRows = append(seasonRows, []any{ season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate, season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag, season.CreatedAt, season.UpdatedAt, }) } if err := copyInsertBatches(ctx, tx, "seasons", []string{ "content_id", "series_id", "season_number", "title", "overview", "air_date", "poster_path", "poster_thumbhash", "metadata_s3_path", "metadata_etag", "created_at", "updated_at", }, seasonRows, func(processed int) { currentWork += processed reportProgress("Importing seasons", currentWork, totalWork) }); err != nil { return nil, err } result.SeasonsCreated = len(seasonRows) episodeRows := make([][]any, 0, len(bundle.Episodes)) episodeIDs := make(map[string]struct{}, len(bundle.Episodes)) for _, episode := range bundle.Episodes { // Skip episodes whose series or season doesn't exist. if _, ok := itemIDs[episode.SeriesID]; !ok { continue } if episode.SeasonID != "" { if _, ok := seasonIDs[episode.SeasonID]; !ok { continue } } episodeIDs[episode.ContentID] = struct{}{} episodeRows = append(episodeRows, []any{ episode.ContentID, episode.SeriesID, nullableString(episode.SeasonID), episode.SeasonNumber, episode.EpisodeNumber, episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB, episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag, episode.CreatedAt, episode.UpdatedAt, }) } if err := copyInsertBatches(ctx, tx, "episodes", []string{ "content_id", "series_id", "season_id", "season_number", "episode_number", "title", "overview", "air_date", "runtime", "rating_imdb", "rating_tmdb", "still_path", "still_thumbhash", "metadata_s3_path", "metadata_etag", "created_at", "updated_at", }, episodeRows, func(processed int) { currentWork += processed reportProgress("Importing episodes", currentWork, totalWork) }); err != nil { return nil, err } result.EpisodesCreated = len(episodeRows) rewrittenFiles := make([]FileRecord, 0, len(bundle.Files)) for _, file := range bundle.Files { // Skip files referencing non-existent items or episodes. if file.ContentID != "" { if _, ok := itemIDs[file.ContentID]; !ok { continue } } if file.EpisodeID != "" { if _, ok := episodeIDs[file.EpisodeID]; !ok { continue } } rewritten, rewriteErr := rewriteFileRecord(file, rewrites, folderPaths) if rewriteErr != nil { return nil, rewriteErr } rewritten.MediaFolderID = folderIDMap[file.MediaFolderID] rewrittenFiles = append(rewrittenFiles, rewritten) } fileRows := make([][]any, 0, len(rewrittenFiles)) for _, file := range rewrittenFiles { videoTracksJSON, jsonErr := json.Marshal(file.VideoTracks) if jsonErr != nil { return nil, fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, jsonErr) } audioTracksJSON, jsonErr := json.Marshal(file.AudioTracks) if jsonErr != nil { return nil, fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, jsonErr) } subtitleTracksJSON, jsonErr := json.Marshal(file.SubtitleTracks) if jsonErr != nil { return nil, fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, jsonErr) } externalSubtitlesJSON, jsonErr := json.Marshal(file.ExternalSubtitles) if jsonErr != nil { return nil, fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, jsonErr) } fileRows = append(fileRows, []any{ nullableString(file.ContentID), nullableString(file.EpisodeID), nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber), file.MediaFolderID, file.FilePath, file.FileSize, nullableString(file.FileHash), file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container, nilIfZero(file.Duration), nilIfZero(file.Bitrate), videoTracksJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON, file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd, nullableString(file.ProbeSource), file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt, }) } if err := copyInsertBatches(ctx, tx, "media_files", []string{ "content_id", "episode_id", "season_number", "episode_number", "media_folder_id", "file_path", "file_size", "file_hash", "codec_video", "codec_audio", "resolution", "audio_channels", "hdr", "container", "duration", "bitrate", "video_tracks", "audio_tracks", "subtitle_tracks", "external_subtitles", "intro_start", "intro_end", "credits_start", "credits_end", "probe_source", "probe_updated_at", "missing_since", "created_at", "updated_at", }, fileRows, func(processed int) { currentWork += processed reportProgress("Importing media files", currentWork, totalWork) }); err != nil { return nil, err } result.FilesCreated = len(rewrittenFiles) linkRows := make([][]any, 0, len(bundle.LibraryLinks)) for _, link := range bundle.LibraryLinks { if _, ok := itemIDs[link.ContentID]; !ok { continue } linkRows = append(linkRows, []any{ link.ContentID, folderIDMap[link.MediaFolderID], link.FirstSeenAt, }) } if err := copyInsertBatches(ctx, tx, "media_item_libraries", []string{"content_id", "media_folder_id", "first_seen_at"}, linkRows, func(processed int) { currentWork += processed reportProgress("Linking items to libraries", currentWork, totalWork) }); err != nil { return nil, err } result.LinksCreated = len(linkRows) reportProgress("Finalizing catalog import", currentWork, totalWork) if err := tx.Commit(ctx); err != nil { return nil, fmt.Errorf("committing catalog seed import: %w", err) } reportProgress("Catalog import completed", totalWork, totalWork) return result, nil } // Batch import items (instead of one-by-one). itemStates, itemsCreated, itemsUpdated, itemsSkipped, err := batchImportItems(ctx, tx, bundle.Items, opts.ConflictMode, func(processed int) { currentWork += processed reportProgress("Importing catalog items", currentWork, totalWork) }) if err != nil { return nil, err } result.ItemsCreated = itemsCreated result.ItemsUpdated = itemsUpdated result.Skipped += itemsSkipped reportProgress("Importing people", currentWork, totalWork) if err := s.replacePeople(ctx, tx, bundle.People, itemStates, opts.ConflictMode, result); err != nil { return nil, fmt.Errorf("replacing people: %w", err) } if err := s.importEmbeddings(ctx, tx, bundle.Embeddings, false, result, func(processed int) { currentWork += processed reportProgress("Importing embeddings", currentWork, totalWork) }); err != nil { return nil, fmt.Errorf("importing embeddings: %w", err) } currentWork++ reportProgress("Importing embeddings", currentWork, totalWork) // Batch import seasons. seasonRows := make([][]any, 0, len(bundle.Seasons)) for _, season := range bundle.Seasons { seasonRows = append(seasonRows, []any{ season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate, season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag, season.CreatedAt, season.UpdatedAt, }) } seasonCounts, err := executeUpsertBatches(ctx, tx, ` INSERT INTO seasons ( content_id, series_id, season_number, title, overview, air_date, poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES `, seasonRows, 12, nil, ` ON CONFLICT (content_id) DO NOTHING`, ` ON CONFLICT (content_id) DO UPDATE SET series_id = EXCLUDED.series_id, season_number = EXCLUDED.season_number, title = EXCLUDED.title, overview = EXCLUDED.overview, air_date = EXCLUDED.air_date, poster_path = EXCLUDED.poster_path, poster_thumbhash = EXCLUDED.poster_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, updated_at = EXCLUDED.updated_at`, opts.ConflictMode, func(processed int) { currentWork += processed reportProgress("Importing seasons", currentWork, totalWork) }) if err != nil { return nil, err } result.SeasonsCreated = seasonCounts.Created result.SeasonsUpdated = seasonCounts.Updated result.Skipped += seasonCounts.Skipped // Batch import episodes. episodeRows := make([][]any, 0, len(bundle.Episodes)) for _, episode := range bundle.Episodes { episodeRows = append(episodeRows, []any{ episode.ContentID, episode.SeriesID, nullableString(episode.SeasonID), episode.SeasonNumber, episode.EpisodeNumber, episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB, episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag, episode.CreatedAt, episode.UpdatedAt, }) } episodeCounts, err := executeUpsertBatches(ctx, tx, ` INSERT INTO episodes ( content_id, series_id, season_id, season_number, episode_number, title, overview, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES `, episodeRows, 17, nil, ` ON CONFLICT (content_id) DO NOTHING`, ` ON CONFLICT (content_id) DO UPDATE SET series_id = EXCLUDED.series_id, season_id = EXCLUDED.season_id, season_number = EXCLUDED.season_number, episode_number = EXCLUDED.episode_number, title = EXCLUDED.title, overview = EXCLUDED.overview, air_date = EXCLUDED.air_date, runtime = EXCLUDED.runtime, rating_imdb = EXCLUDED.rating_imdb, rating_tmdb = EXCLUDED.rating_tmdb, still_path = EXCLUDED.still_path, still_thumbhash = EXCLUDED.still_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, updated_at = EXCLUDED.updated_at`, opts.ConflictMode, func(processed int) { currentWork += processed reportProgress("Importing episodes", currentWork, totalWork) }) if err != nil { return nil, err } result.EpisodesCreated = episodeCounts.Created result.EpisodesUpdated = episodeCounts.Updated result.Skipped += episodeCounts.Skipped // Rewrite file paths and batch import files. rewrittenFiles := make([]FileRecord, 0, len(bundle.Files)) for _, file := range bundle.Files { rewritten, rewriteErr := rewriteFileRecord(file, rewrites, folderPaths) if rewriteErr != nil { return nil, rewriteErr } rewritten.MediaFolderID = folderIDMap[file.MediaFolderID] rewrittenFiles = append(rewrittenFiles, rewritten) } fileRows := make([][]any, 0, len(rewrittenFiles)) for _, file := range rewrittenFiles { videoTracksJSON, jsonErr := json.Marshal(file.VideoTracks) if jsonErr != nil { return nil, fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, jsonErr) } audioTracksJSON, jsonErr := json.Marshal(file.AudioTracks) if jsonErr != nil { return nil, fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, jsonErr) } subtitleTracksJSON, jsonErr := json.Marshal(file.SubtitleTracks) if jsonErr != nil { return nil, fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, jsonErr) } externalSubtitlesJSON, jsonErr := json.Marshal(file.ExternalSubtitles) if jsonErr != nil { return nil, fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, jsonErr) } fileRows = append(fileRows, []any{ nullableString(file.ContentID), nullableString(file.EpisodeID), nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber), file.MediaFolderID, file.FilePath, file.FileSize, nullableString(file.FileHash), file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container, nilIfZero(file.Duration), nilIfZero(file.Bitrate), string(videoTracksJSON), string(audioTracksJSON), string(subtitleTracksJSON), string(externalSubtitlesJSON), file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd, nullableString(file.ProbeSource), file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt, }) } fileCounts, err := executeUpsertBatches(ctx, tx, ` INSERT INTO media_files ( content_id, episode_id, season_number, episode_number, media_folder_id, file_path, file_size, file_hash, codec_video, codec_audio, resolution, audio_channels, hdr, container, duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles, intro_start, intro_end, credits_start, credits_end, probe_source, probe_updated_at, missing_since, created_at, updated_at ) VALUES `, fileRows, 29, map[int]string{16: "::jsonb", 17: "::jsonb", 18: "::jsonb", 19: "::jsonb"}, ` ON CONFLICT (file_path) DO NOTHING`, ` ON CONFLICT (file_path) DO UPDATE SET content_id = EXCLUDED.content_id, episode_id = EXCLUDED.episode_id, season_number = EXCLUDED.season_number, episode_number = EXCLUDED.episode_number, media_folder_id = EXCLUDED.media_folder_id, file_size = EXCLUDED.file_size, file_hash = EXCLUDED.file_hash, codec_video = EXCLUDED.codec_video, codec_audio = EXCLUDED.codec_audio, resolution = EXCLUDED.resolution, audio_channels = EXCLUDED.audio_channels, hdr = EXCLUDED.hdr, container = EXCLUDED.container, duration = EXCLUDED.duration, bitrate = EXCLUDED.bitrate, video_tracks = EXCLUDED.video_tracks, audio_tracks = EXCLUDED.audio_tracks, subtitle_tracks = EXCLUDED.subtitle_tracks, external_subtitles = EXCLUDED.external_subtitles, intro_start = EXCLUDED.intro_start, intro_end = EXCLUDED.intro_end, credits_start = EXCLUDED.credits_start, credits_end = EXCLUDED.credits_end, probe_source = EXCLUDED.probe_source, probe_updated_at = EXCLUDED.probe_updated_at, missing_since = EXCLUDED.missing_since, updated_at = EXCLUDED.updated_at`, opts.ConflictMode, func(processed int) { currentWork += processed reportProgress("Importing media files", currentWork, totalWork) }) if err != nil { return nil, err } result.FilesCreated = fileCounts.Created result.FilesUpdated = fileCounts.Updated result.Skipped += fileCounts.Skipped // Batch import library links. linkRows := make([][]any, 0, len(bundle.LibraryLinks)) for _, link := range bundle.LibraryLinks { linkRows = append(linkRows, []any{link.ContentID, folderIDMap[link.MediaFolderID], link.FirstSeenAt}) } linksAffected, err := executeInsertBatchesCounted(ctx, tx, ` INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at) VALUES `, linkRows, 3, nil, ` ON CONFLICT (content_id, media_folder_id) DO NOTHING`, func(processed int) { currentWork += processed reportProgress("Linking items to libraries", currentWork, totalWork) }) if err != nil { return nil, err } result.LinksCreated = int(linksAffected) reportProgress("Finalizing catalog import", currentWork, totalWork) if err := tx.Commit(ctx); err != nil { return nil, fmt.Errorf("committing catalog seed import: %w", err) } reportProgress("Catalog import completed", totalWork, totalWork) return result, nil } func (s *Service) exportFolders(ctx context.Context, libraryIDs []int) ([]LibraryRecord, error) { folderRepo := catalog.NewFolderRepository(s.pool) var folders []*models.MediaFolder if len(libraryIDs) == 0 { var err error folders, err = folderRepo.List(ctx) if err != nil { return nil, fmt.Errorf("listing folders for export: %w", err) } } else { folders = make([]*models.MediaFolder, 0, len(libraryIDs)) for _, libraryID := range libraryIDs { folder, err := folderRepo.GetByID(ctx, libraryID) if err != nil { return nil, fmt.Errorf("loading folder %d for export: %w", libraryID, err) } folders = append(folders, folder) } } records := make([]LibraryRecord, 0, len(folders)) for _, folder := range folders { paths := append([]string(nil), folder.Paths...) sort.Strings(paths) records = append(records, LibraryRecord{ ExportedID: folder.ID, Paths: paths, Type: folder.Type, Name: folder.Name, Enabled: folder.Enabled, LastScannedAt: folder.LastScannedAt, ScanWarningCode: folder.ScanWarningCode, ScanWarningMessage: folder.ScanWarningMessage, ScanWarningAt: folder.ScanWarningAt, AllowEmptyCleanupOnce: folder.AllowEmptyCleanupOnce, }) } return records, nil } func (s *Service) schemaVersion(ctx context.Context) (int, error) { var version int if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_versions").Scan(&version); err != nil { return 0, fmt.Errorf("loading schema version: %w", err) } return version, nil } func (s *Service) loadSeasons(ctx context.Context, seriesID string) ([]SeasonRecord, error) { rows, err := s.pool.Query(ctx, ` SELECT content_id, series_id, season_number, title, overview, air_date, poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at FROM seasons WHERE series_id = $1 ORDER BY season_number ASC`, seriesID) if err != nil { return nil, fmt.Errorf("querying seasons for %s: %w", seriesID, err) } defer rows.Close() var seasons []SeasonRecord for rows.Next() { var record SeasonRecord if err := rows.Scan( &record.ContentID, &record.SeriesID, &record.SeasonNumber, &record.Title, &record.Overview, &record.AirDate, &record.PosterPath, &record.PosterThumbhash, &record.MetadataS3Path, &record.MetadataEtag, &record.CreatedAt, &record.UpdatedAt, ); err != nil { return nil, fmt.Errorf("scanning season for %s: %w", seriesID, err) } seasons = append(seasons, record) } return seasons, rows.Err() } func (s *Service) loadEpisodes(ctx context.Context, seriesID string) ([]EpisodeRecord, error) { rows, err := s.pool.Query(ctx, ` SELECT content_id, series_id, season_id, season_number, episode_number, title, overview, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at FROM episodes WHERE series_id = $1 ORDER BY season_number ASC, episode_number ASC`, seriesID) if err != nil { return nil, fmt.Errorf("querying episodes for %s: %w", seriesID, err) } defer rows.Close() var episodes []EpisodeRecord for rows.Next() { var record EpisodeRecord var seasonID *string if err := rows.Scan( &record.ContentID, &record.SeriesID, &seasonID, &record.SeasonNumber, &record.EpisodeNumber, &record.Title, &record.Overview, &record.AirDate, &record.Runtime, &record.RatingIMDB, &record.RatingTMDB, &record.StillPath, &record.StillThumbhash, &record.MetadataS3Path, &record.MetadataEtag, &record.CreatedAt, &record.UpdatedAt, ); err != nil { return nil, fmt.Errorf("scanning episode for %s: %w", seriesID, err) } if seasonID != nil { record.SeasonID = *seasonID } episodes = append(episodes, record) } return episodes, rows.Err() } func itemToRecord(item *models.MediaItem) ItemRecord { if strings.TrimSpace(item.SortTitle) == "" { if derived := titleutil.DeriveDefaultSortTitle(item.Title); derived != "" { item.SortTitle = derived } } return ItemRecord{ ContentID: item.ContentID, Type: item.Type, Title: item.Title, SortTitle: item.SortTitle, OriginalTitle: item.OriginalTitle, Year: item.Year, Genres: cloneStrings(item.Genres), ContentRating: item.ContentRating, Runtime: item.Runtime, Overview: item.Overview, Tagline: item.Tagline, RatingIMDB: item.RatingIMDB, RatingTMDB: item.RatingTMDB, RatingRTCritic: item.RatingRTCritic, RatingRTAudience: item.RatingRTAudience, ImdbID: item.ImdbID, TmdbID: item.TmdbID, TvdbID: item.TvdbID, PosterPath: item.PosterPath, PosterThumbhash: item.PosterThumbhash, BackdropPath: item.BackdropPath, BackdropThumbhash: item.BackdropThumbhash, LogoPath: item.LogoPath, MetadataS3Path: item.MetadataS3Path, MetadataEtag: item.MetadataEtag, SeasonCount: item.SeasonCount, Studios: cloneStrings(item.Studios), Networks: cloneStrings(item.Networks), Countries: cloneStrings(item.Countries), ReleaseDate: item.ReleaseDate, FirstAirDate: item.FirstAirDate, LastAirDate: item.LastAirDate, MatchedAt: item.MatchedAt, LastRefreshed: item.LastRefreshed, RefreshFailures: item.RefreshFailures, LockedFields: cloneInts(item.LockedFields), Status: item.Status, CreatedAt: item.CreatedAt, UpdatedAt: item.UpdatedAt, } } func fileToRecord(file *models.MediaFile) FileRecord { return FileRecord{ ContentID: file.ContentID, EpisodeID: file.EpisodeID, SeasonNumber: file.SeasonNumber, EpisodeNumber: file.EpisodeNumber, MediaFolderID: file.MediaFolderID, FilePath: file.FilePath, FileSize: file.FileSize, FileHash: file.FileHash, CodecVideo: file.CodecVideo, CodecAudio: file.CodecAudio, Resolution: file.Resolution, AudioChannels: file.AudioChannels, HDR: file.HDR, Container: file.Container, Duration: file.Duration, Bitrate: file.Bitrate, VideoTracks: toVideoTrackRecords(file.VideoTracks), AudioTracks: toAudioTrackRecords(file.AudioTracks), SubtitleTracks: toSubtitleTrackRecords(file.SubtitleTracks), ExternalSubtitles: toExternalSubtitleRecords(file.ExternalSubtitles), IntroStart: file.IntroStart, IntroEnd: file.IntroEnd, CreditsStart: file.CreditsStart, CreditsEnd: file.CreditsEnd, ProbeSource: file.ProbeSource, ProbeUpdatedAt: file.ProbeUpdatedAt, MissingSince: file.MissingSince, CreatedAt: file.CreatedAt, UpdatedAt: file.UpdatedAt, } } func encodeBundle(bundle Bundle) ([]byte, error) { var raw bytes.Buffer gz := gzip.NewWriter(&raw) if err := json.NewEncoder(gz).Encode(bundle); err != nil { _ = gz.Close() return nil, fmt.Errorf("encoding catalog seed bundle: %w", err) } if err := gz.Close(); err != nil { return nil, fmt.Errorf("closing catalog seed bundle: %w", err) } return raw.Bytes(), nil } func decodeBundle(data []byte) (*Bundle, error) { gz, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("%w: opening catalog seed bundle: %v", ErrInvalidBundle, err) } defer gz.Close() payload, err := io.ReadAll(gz) if err != nil { return nil, fmt.Errorf("%w: reading catalog seed bundle: %v", ErrInvalidBundle, err) } var bundle Bundle if err := json.Unmarshal(payload, &bundle); err != nil { return nil, fmt.Errorf("%w: decoding catalog seed bundle: %v", ErrInvalidBundle, err) } return &bundle, nil } func prepareLibraries(records []LibraryRecord, rewrites []PathRewrite) ([]LibraryRecord, map[int][]string, []string, error) { libraries := make([]LibraryRecord, 0, len(records)) folderPaths := make(map[int][]string, len(records)) var unmatched []string for _, record := range records { finalPaths := make([]string, 0, len(record.Paths)) for _, path := range record.Paths { rewritten, _ := rewritePath(path, rewrites) rewritten = normalizePath(rewritten) if !filepath.IsAbs(rewritten) { return nil, nil, nil, fmt.Errorf("catalog seed path %q is not absolute after rewrite", rewritten) } finalPaths = append(finalPaths, rewritten) if _, err := os.Stat(rewritten); err != nil { if os.IsNotExist(err) { unmatched = append(unmatched, rewritten) continue } return nil, nil, nil, fmt.Errorf("checking library path %q: %w", rewritten, err) } } sort.Strings(finalPaths) record.Paths = dedupeStrings(finalPaths) folderPaths[record.ExportedID] = append([]string(nil), record.Paths...) libraries = append(libraries, record) } return libraries, folderPaths, dedupeStrings(unmatched), nil } func validateFileRoots(files []FileRecord, folderPaths map[int][]string, rewrites []PathRewrite) error { var unmatched []string for _, file := range files { rewritten, matched := rewritePath(file.FilePath, rewrites) rewritten = normalizePath(rewritten) paths := folderPaths[file.MediaFolderID] ok := false for _, root := range paths { if hasPathPrefix(rewritten, root) { ok = true break } } if !ok { if matched { unmatched = append(unmatched, rewritten) } else { unmatched = append(unmatched, file.FilePath) } } } if len(unmatched) > 0 { sort.Strings(unmatched) return &UnmatchedRootsError{Roots: dedupeStrings(unmatched)} } return nil } func rewriteFileRecord(file FileRecord, rewrites []PathRewrite, folderPaths map[int][]string) (FileRecord, error) { rewrittenPath, _ := rewritePath(file.FilePath, rewrites) file.FilePath = normalizePath(rewrittenPath) if !filepath.IsAbs(file.FilePath) { return FileRecord{}, fmt.Errorf("catalog seed file path %q is not absolute after rewrite", file.FilePath) } for i := range file.ExternalSubtitles { if file.ExternalSubtitles[i].Path == "" { continue } subtitlePath, _ := rewritePath(file.ExternalSubtitles[i].Path, rewrites) file.ExternalSubtitles[i].Path = normalizePath(subtitlePath) if !filepath.IsAbs(file.ExternalSubtitles[i].Path) { return FileRecord{}, fmt.Errorf("catalog seed subtitle path %q is not absolute after rewrite", file.ExternalSubtitles[i].Path) } } paths := folderPaths[file.MediaFolderID] for _, root := range paths { if hasPathPrefix(file.FilePath, root) { return file, nil } } return FileRecord{}, &UnmatchedRootsError{Roots: []string{file.FilePath}} } func importLibrary(ctx context.Context, tx pgx.Tx, library LibraryRecord, overwrite bool) (localID int, created bool, matched bool, err error) { existingIDs, err := lookupFolderIDsByPaths(ctx, tx, library.Paths) if err != nil { return 0, false, false, err } switch len(existingIDs) { case 0: err = tx.QueryRow(ctx, ` INSERT INTO media_folders ( type, name, enabled, last_scanned_at, scan_warning_code, scan_warning_message, scan_warning_at, allow_empty_cleanup_once ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, library.Type, library.Name, library.Enabled, library.LastScannedAt, library.ScanWarningCode, library.ScanWarningMessage, library.ScanWarningAt, library.AllowEmptyCleanupOnce, ).Scan(&localID) if err != nil { return 0, false, false, fmt.Errorf("creating imported library %q: %w", library.Name, err) } for _, path := range library.Paths { if _, err := tx.Exec(ctx, `INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2)`, localID, path, ); err != nil { return 0, false, false, fmt.Errorf("creating imported library path %q: %w", path, err) } } return localID, true, false, nil case 1: localID = existingIDs[0] if overwrite { if _, err := tx.Exec(ctx, ` UPDATE media_folders SET type = $1, name = $2, enabled = $3, last_scanned_at = $4, scan_warning_code = $5, scan_warning_message = $6, scan_warning_at = $7, allow_empty_cleanup_once = $8 WHERE id = $9`, library.Type, library.Name, library.Enabled, library.LastScannedAt, library.ScanWarningCode, library.ScanWarningMessage, library.ScanWarningAt, library.AllowEmptyCleanupOnce, localID, ); err != nil { return 0, false, false, fmt.Errorf("updating imported library %d: %w", localID, err) } } for _, path := range library.Paths { if _, err := tx.Exec(ctx, `INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2) ON CONFLICT (path) DO NOTHING`, localID, path, ); err != nil { return 0, false, false, fmt.Errorf("upserting library path %q: %w", path, err) } } return localID, false, true, nil default: return 0, false, false, fmt.Errorf("catalog seed paths for library %q map to multiple local libraries", library.Name) } } const catalogImportBatchSize = 1000 type importPersonState struct { ID int64 Name string LowerName string TmdbID string ImdbID string TvdbID string PlexGUID string PhotoPath string PhotoThumbhash string IsNew bool NeedsUpdate bool } func isEmptyCatalogImportTarget(ctx context.Context, tx pgx.Tx) (bool, error) { var empty bool err := tx.QueryRow(ctx, ` SELECT NOT EXISTS (SELECT 1 FROM media_folders) AND NOT EXISTS (SELECT 1 FROM media_items) AND NOT EXISTS (SELECT 1 FROM item_people) AND NOT EXISTS (SELECT 1 FROM seasons) AND NOT EXISTS (SELECT 1 FROM episodes) AND NOT EXISTS (SELECT 1 FROM media_files) AND NOT EXISTS (SELECT 1 FROM media_item_libraries)`). Scan(&empty) if err != nil { return false, fmt.Errorf("checking empty catalog import target: %w", err) } return empty, nil } func bulkInsertItems(ctx context.Context, tx pgx.Tx, items []ItemRecord, onBatch func(processed int)) error { rows := make([][]any, 0, len(items)) for _, item := range items { rows = append(rows, []any{ item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres, item.ContentRating, item.Runtime, item.Overview, item.Tagline, item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience, item.ImdbID, item.TmdbID, item.TvdbID, item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath, item.MetadataS3Path, item.MetadataEtag, item.SeasonCount, item.Studios, item.Networks, item.Countries, item.Keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate, item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status, item.CreatedAt, item.UpdatedAt, }) } return executeInsertBatches(ctx, tx, ` INSERT INTO media_items ( content_id, type, title, sort_title, original_title, year, genres, content_rating, runtime, overview, tagline, rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience, imdb_id, tmdb_id, tvdb_id, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, metadata_s3_path, metadata_etag, season_count, studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date, matched_at, last_refreshed, refresh_failures, locked_fields, status, created_at, updated_at ) VALUES `, rows, 41, nil, "", onBatch, ) } func (s *Service) replacePeople(ctx context.Context, tx pgx.Tx, people []PersonRecord, itemStates map[string]bool, mode ConflictMode, result *ImportResult) error { shouldReplace := make(map[string]struct{}) if itemStates == nil { // Empty-catalog fast path: replace everything. for _, p := range people { shouldReplace[p.ContentID] = struct{}{} } } else { for contentID, changed := range itemStates { if mode == ConflictModeOverwrite || changed { shouldReplace[contentID] = struct{}{} } } } if len(shouldReplace) == 0 { return nil } contentIDs := mapKeys(shouldReplace) sort.Strings(contentIDs) if err := clearImportedPeople(ctx, tx, contentIDs); err != nil { return err } filtered := make([]PersonRecord, 0, len(people)) tmdbIDs := make([]string, 0, len(people)) imdbIDs := make([]string, 0, len(people)) nameKeys := make([]string, 0, len(people)) for _, rec := range people { if _, ok := shouldReplace[rec.ContentID]; !ok { continue } filtered = append(filtered, rec) if rec.TmdbID != "" { tmdbIDs = append(tmdbIDs, rec.TmdbID) } if rec.ImdbID != "" { imdbIDs = append(imdbIDs, rec.ImdbID) } if rec.Name != "" { nameKeys = append(nameKeys, strings.ToLower(rec.Name)) } } if len(filtered) == 0 { result.CreditsReplaced += len(contentIDs) return nil } byTMDB, byIMDb, byName, err := loadExistingImportPeople(ctx, tx, tmdbIDs, imdbIDs, nameKeys) if err != nil { return err } newPeople := make([]*importPersonState, 0) updatedPeople := make(map[int64]*importPersonState) type itemPersonKey struct { contentID string personID int64 kind int character string } seenItemPeople := make(map[itemPersonKey]struct{}, len(filtered)) itemPeopleRows := make([][]any, 0, len(filtered)) for _, rec := range filtered { person, resolveErr := resolveImportPerson(rec, byTMDB, byIMDb, byName, &newPeople, updatedPeople) if resolveErr != nil { return fmt.Errorf("resolving person %q for %s: %w", rec.Name, rec.ContentID, resolveErr) } // PostgreSQL's ON CONFLICT cannot handle the same conflict-target tuple // appearing twice in one INSERT. Deduplicate after person resolution so // that collisions caused by person-merging are also caught. key := itemPersonKey{rec.ContentID, person.ID, rec.Kind, rec.Character} if _, exists := seenItemPeople[key]; exists { continue } seenItemPeople[key] = struct{}{} itemPersonID, idErr := nextInt64ID() if idErr != nil { return fmt.Errorf("generating item_people id for %s: %w", rec.ContentID, idErr) } itemPeopleRows = append(itemPeopleRows, []any{ itemPersonID, rec.ContentID, person.ID, rec.Kind, rec.Character, rec.SortOrder, }) } if err := bulkInsertPeople(ctx, tx, newPeople); err != nil { return err } if err := bulkUpdatePeople(ctx, tx, updatedPeople); err != nil { return err } if err := bulkInsertItemPeople(ctx, tx, itemPeopleRows); err != nil { return err } result.CreditsReplaced += len(contentIDs) return nil } // embeddingImportBatchSize is larger than the general batch size because // embeddings only have 4 columns (4 × 5000 = 20,000 params, well within the 65,535 limit). const embeddingImportBatchSize = 5000 func (s *Service) importEmbeddings(ctx context.Context, tx pgx.Tx, embeddings []EmbeddingRecord, emptyTarget bool, result *ImportResult, onBatch func(int)) error { if len(embeddings) == 0 { return nil } // For empty targets, drop the HNSW index before bulk loading and recreate after. // Maintaining an HNSW graph incrementally during inserts is far slower than building // it once on the complete dataset. if emptyTarget { if _, err := tx.Exec(ctx, `DROP INDEX IF EXISTS idx_media_item_embeddings_hnsw`); err != nil { return fmt.Errorf("dropping embedding index for bulk load: %w", err) } } rows := make([][]any, 0, len(embeddings)) for _, rec := range embeddings { padded := rec.Embedding if len(padded) > recommendations.CanonicalEmbeddingDimensions { return fmt.Errorf( "embedding vector length %d exceeds canonical dimension %d for media item %s", len(padded), recommendations.CanonicalEmbeddingDimensions, rec.MediaItemID, ) } if len(padded) < recommendations.CanonicalEmbeddingDimensions { p := make([]float32, recommendations.CanonicalEmbeddingDimensions) copy(p, padded) padded = p } rows = append(rows, []any{ rec.MediaItemID, pgvector.NewVector(padded), rec.Model, rec.CanonicalText, }) } batchCallback := func(processed int) { result.EmbeddingsImported += processed if onBatch != nil { onBatch(processed) } } var suffix string if emptyTarget { suffix = "" } else { suffix = ` ON CONFLICT (media_item_id) DO UPDATE SET embedding = EXCLUDED.embedding, model = EXCLUDED.model, canonical_text = EXCLUDED.canonical_text, updated_at = NOW()` } for start := 0; start < len(rows); start += embeddingImportBatchSize { end := min(start+embeddingImportBatchSize, len(rows)) var builder strings.Builder builder.WriteString(` INSERT INTO media_item_embeddings (media_item_id, embedding, model, canonical_text) VALUES `) args := make([]any, 0, (end-start)*4) for rowIndex := start; rowIndex < end; rowIndex++ { if rowIndex > start { builder.WriteString(", ") } builder.WriteString(buildValuesClause(len(args)+1, 4, nil)) args = append(args, rows[rowIndex]...) } builder.WriteString(suffix) if _, err := tx.Exec(ctx, builder.String(), args...); err != nil { return fmt.Errorf("inserting embeddings batch: %w", err) } batchCallback(end - start) } // Recreate the HNSW index after bulk load. if emptyTarget { if _, err := tx.Exec(ctx, fmt.Sprintf( `CREATE INDEX idx_media_item_embeddings_hnsw ON media_item_embeddings USING hnsw ((embedding::halfvec(%d)) halfvec_cosine_ops)`, recommendations.CanonicalEmbeddingDimensions, )); err != nil { return fmt.Errorf("recreating embedding index after bulk load: %w", err) } } return nil } func bulkInsertSeasons(ctx context.Context, tx pgx.Tx, seasons []SeasonRecord, onBatch func(processed int)) error { rows := make([][]any, 0, len(seasons)) for _, season := range seasons { rows = append(rows, []any{ season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate, season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag, season.CreatedAt, season.UpdatedAt, }) } return executeInsertBatches(ctx, tx, ` INSERT INTO seasons ( content_id, series_id, season_number, title, overview, air_date, poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES `, rows, 12, nil, "", onBatch, ) } func bulkInsertEpisodes(ctx context.Context, tx pgx.Tx, episodes []EpisodeRecord, onBatch func(processed int)) error { rows := make([][]any, 0, len(episodes)) for _, episode := range episodes { rows = append(rows, []any{ episode.ContentID, episode.SeriesID, nullableString(episode.SeasonID), episode.SeasonNumber, episode.EpisodeNumber, episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB, episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag, episode.CreatedAt, episode.UpdatedAt, }) } return executeInsertBatches(ctx, tx, ` INSERT INTO episodes ( content_id, series_id, season_id, season_number, episode_number, title, overview, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES `, rows, 17, nil, "", onBatch, ) } func bulkInsertFiles(ctx context.Context, tx pgx.Tx, files []FileRecord, onBatch func(processed int)) error { rows := make([][]any, 0, len(files)) for _, file := range files { videoTracksJSON, err := json.Marshal(file.VideoTracks) if err != nil { return fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, err) } audioTracksJSON, err := json.Marshal(file.AudioTracks) if err != nil { return fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, err) } subtitleTracksJSON, err := json.Marshal(file.SubtitleTracks) if err != nil { return fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, err) } externalSubtitlesJSON, err := json.Marshal(file.ExternalSubtitles) if err != nil { return fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, err) } rows = append(rows, []any{ nullableString(file.ContentID), nullableString(file.EpisodeID), nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber), file.MediaFolderID, file.FilePath, file.FileSize, nullableString(file.FileHash), file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container, nilIfZero(file.Duration), nilIfZero(file.Bitrate), string(videoTracksJSON), string(audioTracksJSON), string(subtitleTracksJSON), string(externalSubtitlesJSON), file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd, nullableString(file.ProbeSource), file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt, }) } return executeInsertBatches(ctx, tx, ` INSERT INTO media_files ( content_id, episode_id, season_number, episode_number, media_folder_id, file_path, file_size, file_hash, codec_video, codec_audio, resolution, audio_channels, hdr, container, duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles, intro_start, intro_end, credits_start, credits_end, probe_source, probe_updated_at, missing_since, created_at, updated_at ) VALUES `, rows, 29, map[int]string{ 16: "::jsonb", 17: "::jsonb", 18: "::jsonb", 19: "::jsonb", }, "", onBatch, ) } func bulkInsertLibraryLinks(ctx context.Context, tx pgx.Tx, links []LibraryLinkRecord, onBatch func(processed int)) error { rows := make([][]any, 0, len(links)) for _, link := range links { rows = append(rows, []any{link.ContentID, link.MediaFolderID, link.FirstSeenAt}) } return executeInsertBatches(ctx, tx, ` INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at) VALUES `, rows, 3, nil, "", onBatch, ) } func executeInsertBatches(ctx context.Context, tx pgx.Tx, prefix string, rows [][]any, columnsPerRow int, casts map[int]string, suffix string, onBatch func(processed int)) error { _, err := executeInsertBatchesCounted(ctx, tx, prefix, rows, columnsPerRow, casts, suffix, onBatch) return err } // executeInsertBatchesCounted runs batched inserts and returns total rows affected. // For ON CONFLICT DO NOTHING, RowsAffected = rows actually inserted. // For ON CONFLICT DO UPDATE, RowsAffected = all rows (inserts + updates). func executeInsertBatchesCounted(ctx context.Context, tx pgx.Tx, prefix string, rows [][]any, columnsPerRow int, casts map[int]string, suffix string, onBatch func(processed int)) (int64, error) { if len(rows) == 0 { return 0, nil } var total int64 for start := 0; start < len(rows); start += catalogImportBatchSize { end := min(start+catalogImportBatchSize, len(rows)) var builder strings.Builder builder.WriteString(prefix) args := make([]any, 0, (end-start)*columnsPerRow) for rowIndex := start; rowIndex < end; rowIndex++ { if rowIndex > start { builder.WriteString(", ") } builder.WriteString(buildValuesClause(len(args)+1, columnsPerRow, casts)) args = append(args, rows[rowIndex]...) } builder.WriteString(suffix) tag, err := tx.Exec(ctx, builder.String(), args...) if err != nil { return total, err } total += tag.RowsAffected() if onBatch != nil { onBatch(end - start) } } return total, nil } type batchUpsertCounts struct { Created int Updated int Skipped int } // executeUpsertBatches runs batched inserts with ON CONFLICT handling and returns // accurate created/updated/skipped counts. For skip_existing mode it uses Exec // with RowsAffected; for overwrite mode it uses RETURNING (xmax = 0) to // distinguish inserts from updates. func executeUpsertBatches(ctx context.Context, tx pgx.Tx, prefix string, rows [][]any, columnsPerRow int, casts map[int]string, skipSuffix, upsertSuffix string, mode ConflictMode, onBatch func(processed int)) (batchUpsertCounts, error) { var counts batchUpsertCounts if len(rows) == 0 { return counts, nil } if mode == ConflictModeSkipExisting { affected, err := executeInsertBatchesCounted(ctx, tx, prefix, rows, columnsPerRow, casts, skipSuffix, onBatch) if err != nil { return counts, err } counts.Created = int(affected) counts.Skipped = len(rows) - counts.Created return counts, nil } // Overwrite mode: append RETURNING (xmax = 0) to distinguish inserts from updates. fullSuffix := upsertSuffix + ` RETURNING (xmax = 0)` for start := 0; start < len(rows); start += catalogImportBatchSize { end := min(start+catalogImportBatchSize, len(rows)) var builder strings.Builder builder.WriteString(prefix) args := make([]any, 0, (end-start)*columnsPerRow) for rowIndex := start; rowIndex < end; rowIndex++ { if rowIndex > start { builder.WriteString(", ") } builder.WriteString(buildValuesClause(len(args)+1, columnsPerRow, casts)) args = append(args, rows[rowIndex]...) } builder.WriteString(fullSuffix) queryRows, queryErr := tx.Query(ctx, builder.String(), args...) if queryErr != nil { return counts, queryErr } for queryRows.Next() { var isNew bool if scanErr := queryRows.Scan(&isNew); scanErr != nil { queryRows.Close() return counts, scanErr } if isNew { counts.Created++ } else { counts.Updated++ } } queryRows.Close() if err := queryRows.Err(); err != nil { return counts, err } if onBatch != nil { onBatch(end - start) } } return counts, nil } const copyBatchSize = 10000 // copyInsertBatches uses PostgreSQL's COPY protocol for fast bulk loading. // Much faster than multi-row INSERT as it bypasses query parsing/planning. // Only suitable for empty tables (no ON CONFLICT support). func copyInsertBatches(ctx context.Context, tx pgx.Tx, table string, columns []string, rows [][]any, onBatch func(int)) error { if len(rows) == 0 { return nil } for start := 0; start < len(rows); start += copyBatchSize { end := min(start+copyBatchSize, len(rows)) _, err := tx.CopyFrom(ctx, pgx.Identifier{table}, columns, pgx.CopyFromRows(rows[start:end]), ) if err != nil { return fmt.Errorf("copy into %s: %w", table, err) } if onBatch != nil { onBatch(end - start) } } return nil } func buildValuesClause(base, columns int, casts map[int]string) string { parts := make([]string, 0, columns) for i := range columns { part := fmt.Sprintf("$%d", base+i) if cast, ok := casts[i]; ok { part += cast } parts = append(parts, part) } return "(" + strings.Join(parts, ", ") + ")" } func loadExistingImportPeople(ctx context.Context, tx pgx.Tx, tmdbIDs, imdbIDs, nameKeys []string) (map[string]*importPersonState, map[string]*importPersonState, map[string]*importPersonState, error) { byTMDB := make(map[string]*importPersonState) byIMDb := make(map[string]*importPersonState) byName := make(map[string]*importPersonState) statesByID := make(map[int64]*importPersonState) register := func(state importPersonState) { canonical := statesByID[state.ID] if canonical == nil { copied := state canonical = &copied statesByID[state.ID] = canonical } if canonical.Name == "" { canonical.Name = state.Name canonical.LowerName = state.LowerName } if canonical.TmdbID == "" { canonical.TmdbID = state.TmdbID } if canonical.ImdbID == "" { canonical.ImdbID = state.ImdbID } if canonical.TvdbID == "" { canonical.TvdbID = state.TvdbID } if canonical.PlexGUID == "" { canonical.PlexGUID = state.PlexGUID } if canonical.PhotoPath == "" { canonical.PhotoPath = state.PhotoPath } if canonical.PhotoThumbhash == "" { canonical.PhotoThumbhash = state.PhotoThumbhash } registerImportPerson(canonical, byTMDB, byIMDb, byName) } if err := queryImportPeople(ctx, tx, ` SELECT id, name, LOWER(name), tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash FROM people WHERE tmdb_id = ANY($1)`, dedupeStrings(tmdbIDs), register, ); err != nil { return nil, nil, nil, err } if err := queryImportPeople(ctx, tx, ` SELECT id, name, LOWER(name), tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash FROM people WHERE imdb_id = ANY($1)`, dedupeStrings(imdbIDs), register, ); err != nil { return nil, nil, nil, err } if err := queryImportPeople(ctx, tx, ` SELECT DISTINCT ON (LOWER(name)) id, name, LOWER(name), tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash FROM people WHERE LOWER(name) = ANY($1) ORDER BY LOWER(name), id`, dedupeStrings(nameKeys), register, ); err != nil { return nil, nil, nil, err } return byTMDB, byIMDb, byName, nil } func queryImportPeople(ctx context.Context, tx pgx.Tx, query string, keys []string, add func(importPersonState)) error { if len(keys) == 0 { return nil } rows, err := tx.Query(ctx, query, keys) if err != nil { return fmt.Errorf("loading imported people matches: %w", err) } defer rows.Close() for rows.Next() { var state importPersonState if err := rows.Scan( &state.ID, &state.Name, &state.LowerName, &state.TmdbID, &state.ImdbID, &state.TvdbID, &state.PlexGUID, &state.PhotoPath, &state.PhotoThumbhash, ); err != nil { return fmt.Errorf("scanning imported person match: %w", err) } add(state) } if err := rows.Err(); err != nil { return fmt.Errorf("iterating imported person matches: %w", err) } return nil } func resolveImportPerson(rec PersonRecord, byTMDB, byIMDb, byName map[string]*importPersonState, newPeople *[]*importPersonState, updatedPeople map[int64]*importPersonState) (*importPersonState, error) { nameKey := strings.ToLower(rec.Name) person := findImportPerson(rec, nameKey, byTMDB, byIMDb, byName) if person == nil { personID, err := nextInt64ID() if err != nil { return nil, err } person = &importPersonState{ ID: personID, Name: rec.Name, LowerName: nameKey, TmdbID: rec.TmdbID, ImdbID: rec.ImdbID, TvdbID: rec.TvdbID, PlexGUID: rec.PlexGUID, PhotoPath: rec.PhotoPath, PhotoThumbhash: rec.PhotoThumbhash, IsNew: true, } *newPeople = append(*newPeople, person) registerImportPerson(person, byTMDB, byIMDb, byName) return person, nil } enrichImportPerson(person, rec) registerImportPerson(person, byTMDB, byIMDb, byName) if person.NeedsUpdate { updatedPeople[person.ID] = person } return person, nil } func findImportPerson(rec PersonRecord, nameKey string, byTMDB, byIMDb, byName map[string]*importPersonState) *importPersonState { if rec.TmdbID != "" { if person := byTMDB[rec.TmdbID]; person != nil { return person } } if rec.ImdbID != "" { if person := byIMDb[rec.ImdbID]; person != nil { return person } } if nameKey != "" { if person := byName[nameKey]; person != nil { return person } } return nil } func enrichImportPerson(person *importPersonState, rec PersonRecord) { if person.TmdbID == "" && rec.TmdbID != "" { person.TmdbID = rec.TmdbID person.NeedsUpdate = person.NeedsUpdate || !person.IsNew } if person.ImdbID == "" && rec.ImdbID != "" { person.ImdbID = rec.ImdbID person.NeedsUpdate = person.NeedsUpdate || !person.IsNew } if person.TvdbID == "" && rec.TvdbID != "" { person.TvdbID = rec.TvdbID person.NeedsUpdate = person.NeedsUpdate || !person.IsNew } if person.PlexGUID == "" && rec.PlexGUID != "" { person.PlexGUID = rec.PlexGUID person.NeedsUpdate = person.NeedsUpdate || !person.IsNew } if person.PhotoPath == "" && rec.PhotoPath != "" { person.PhotoPath = rec.PhotoPath person.NeedsUpdate = person.NeedsUpdate || !person.IsNew } if person.PhotoThumbhash == "" && rec.PhotoThumbhash != "" { person.PhotoThumbhash = rec.PhotoThumbhash person.NeedsUpdate = person.NeedsUpdate || !person.IsNew } } func registerImportPerson(person *importPersonState, byTMDB, byIMDb, byName map[string]*importPersonState) { if person.TmdbID != "" { byTMDB[person.TmdbID] = person } if person.ImdbID != "" { byIMDb[person.ImdbID] = person } if person.LowerName != "" { byName[person.LowerName] = person } } func clearImportedPeople(ctx context.Context, tx pgx.Tx, contentIDs []string) error { if len(contentIDs) == 0 { return nil } if _, err := tx.Exec(ctx, `DELETE FROM item_people WHERE content_id = ANY($1)`, contentIDs); err != nil { return fmt.Errorf("clearing imported people: %w", err) } return nil } func bulkInsertPeople(ctx context.Context, tx pgx.Tx, people []*importPersonState) error { rows := make([][]any, 0, len(people)) for _, person := range people { rows = append(rows, []any{ person.ID, person.Name, "", "", nil, nil, "", "", person.PhotoPath, person.PhotoThumbhash, person.TmdbID, person.ImdbID, person.TvdbID, person.PlexGUID, }) } if err := executeInsertBatches(ctx, tx, ` INSERT INTO people ( id, name, sort_name, bio, birth_date, death_date, birthplace, homepage, photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid ) VALUES `, rows, 14, nil, "", nil, ); err != nil { return fmt.Errorf("inserting imported people: %w", err) } return nil } func bulkUpdatePeople(ctx context.Context, tx pgx.Tx, people map[int64]*importPersonState) error { if len(people) == 0 { return nil } ids := make([]int64, 0, len(people)) for id := range people { ids = append(ids, id) } sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) rows := make([][]any, 0, len(ids)) for _, id := range ids { person := people[id] rows = append(rows, []any{ person.ID, person.TmdbID, person.ImdbID, person.TvdbID, person.PlexGUID, person.PhotoPath, person.PhotoThumbhash, }) } if err := executeInsertBatches(ctx, tx, ` UPDATE people AS p SET tmdb_id = v.tmdb_id, imdb_id = v.imdb_id, tvdb_id = v.tvdb_id, plex_guid = v.plex_guid, photo_path = v.photo_path, photo_thumbhash = v.photo_thumbhash, updated_at = NOW() FROM (VALUES `, rows, 7, nil, `) AS v(id, tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash) WHERE p.id = v.id`, nil, ); err != nil { return fmt.Errorf("updating imported people: %w", err) } return nil } func bulkInsertItemPeople(ctx context.Context, tx pgx.Tx, rows [][]any) error { if err := executeInsertBatches(ctx, tx, ` INSERT INTO item_people (id, content_id, person_id, kind, character, sort_order) VALUES `, rows, 6, nil, ` ON CONFLICT (content_id, person_id, kind, character) DO UPDATE SET sort_order = EXCLUDED.sort_order`, nil, ); err != nil { return fmt.Errorf("inserting imported item_people: %w", err) } return nil } func nextInt64ID() (int64, error) { idStr, err := idgen.NextID() if err != nil { return 0, err } id, err := strconv.ParseInt(idStr, 10, 64) if err != nil { return 0, err } return id, nil } func lookupFolderIDsByPaths(ctx context.Context, tx pgx.Tx, paths []string) ([]int, error) { if len(paths) == 0 { return nil, nil } rows, err := tx.Query(ctx, `SELECT DISTINCT media_folder_id FROM media_folder_paths WHERE path = ANY($1)`, paths) if err != nil { return nil, fmt.Errorf("loading existing folder paths: %w", err) } defer rows.Close() var ids []int for rows.Next() { var id int if err := rows.Scan(&id); err != nil { return nil, fmt.Errorf("scanning existing folder id: %w", err) } ids = append(ids, id) } sort.Ints(ids) return ids, rows.Err() } // batchImportItems inserts items in multi-row batches with ON CONFLICT handling. // Returns a map of content_id → changed (true if created or updated) for people import, // plus aggregate created/updated/skipped counts. func batchImportItems(ctx context.Context, tx pgx.Tx, items []ItemRecord, mode ConflictMode, onBatch func(processed int)) (itemStates map[string]bool, created, updated, skipped int, err error) { itemStates = make(map[string]bool, len(items)) if len(items) == 0 { return itemStates, 0, 0, 0, nil } rows := make([][]any, 0, len(items)) contentIDs := make([]string, 0, len(items)) for _, item := range items { studios, networks, countries, keywords := itemRecordStringArrays(item) contentIDs = append(contentIDs, item.ContentID) rows = append(rows, []any{ item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres, item.ContentRating, item.Runtime, item.Overview, item.Tagline, item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience, item.ImdbID, item.TmdbID, item.TvdbID, item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath, item.MetadataS3Path, item.MetadataEtag, item.SeasonCount, studios, networks, countries, keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate, item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status, item.CreatedAt, item.UpdatedAt, }) } const colCount = 41 prefix := ` INSERT INTO media_items ( content_id, type, title, sort_title, original_title, year, genres, content_rating, runtime, overview, tagline, rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience, imdb_id, tmdb_id, tvdb_id, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, metadata_s3_path, metadata_etag, season_count, studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date, matched_at, last_refreshed, refresh_failures, locked_fields, status, created_at, updated_at ) VALUES ` var suffix string if mode == ConflictModeSkipExisting { suffix = ` ON CONFLICT (content_id) DO NOTHING RETURNING content_id` } else { suffix = ` ON CONFLICT (content_id) DO UPDATE SET type = EXCLUDED.type, title = EXCLUDED.title, sort_title = EXCLUDED.sort_title, original_title = EXCLUDED.original_title, year = EXCLUDED.year, genres = EXCLUDED.genres, content_rating = EXCLUDED.content_rating, runtime = EXCLUDED.runtime, overview = EXCLUDED.overview, tagline = EXCLUDED.tagline, rating_imdb = EXCLUDED.rating_imdb, rating_tmdb = EXCLUDED.rating_tmdb, rating_rt_critic = EXCLUDED.rating_rt_critic, rating_rt_audience = EXCLUDED.rating_rt_audience, imdb_id = EXCLUDED.imdb_id, tmdb_id = EXCLUDED.tmdb_id, tvdb_id = EXCLUDED.tvdb_id, poster_path = EXCLUDED.poster_path, poster_thumbhash = EXCLUDED.poster_thumbhash, backdrop_path = EXCLUDED.backdrop_path, backdrop_thumbhash = EXCLUDED.backdrop_thumbhash, logo_path = EXCLUDED.logo_path, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, season_count = EXCLUDED.season_count, studios = EXCLUDED.studios, networks = EXCLUDED.networks, countries = EXCLUDED.countries, keywords = EXCLUDED.keywords, original_language = EXCLUDED.original_language, release_date = EXCLUDED.release_date, first_air_date = EXCLUDED.first_air_date, last_air_date = EXCLUDED.last_air_date, matched_at = EXCLUDED.matched_at, last_refreshed = EXCLUDED.last_refreshed, refresh_failures = EXCLUDED.refresh_failures, locked_fields = EXCLUDED.locked_fields, status = EXCLUDED.status, updated_at = EXCLUDED.updated_at RETURNING content_id, (xmax = 0)` } for start := 0; start < len(rows); start += catalogImportBatchSize { end := min(start+catalogImportBatchSize, len(rows)) var builder strings.Builder builder.WriteString(prefix) args := make([]any, 0, (end-start)*colCount) for rowIndex := start; rowIndex < end; rowIndex++ { if rowIndex > start { builder.WriteString(", ") } builder.WriteString(buildValuesClause(len(args)+1, colCount, nil)) args = append(args, rows[rowIndex]...) } builder.WriteString(suffix) queryRows, queryErr := tx.Query(ctx, builder.String(), args...) if queryErr != nil { return nil, 0, 0, 0, fmt.Errorf("batch importing items: %w", queryErr) } if mode == ConflictModeSkipExisting { // RETURNING content_id — only returned for actually-inserted rows. createdSet := make(map[string]struct{}) for queryRows.Next() { var contentID string if scanErr := queryRows.Scan(&contentID); scanErr != nil { queryRows.Close() return nil, 0, 0, 0, fmt.Errorf("scanning imported item id: %w", scanErr) } createdSet[contentID] = struct{}{} } queryRows.Close() if queryErr = queryRows.Err(); queryErr != nil { return nil, 0, 0, 0, fmt.Errorf("iterating imported items: %w", queryErr) } for _, cid := range contentIDs[start:end] { if _, ok := createdSet[cid]; ok { itemStates[cid] = true created++ } else { skipped++ } } } else { // RETURNING content_id, (xmax = 0) — returns for every row. for queryRows.Next() { var contentID string var isNew bool if scanErr := queryRows.Scan(&contentID, &isNew); scanErr != nil { queryRows.Close() return nil, 0, 0, 0, fmt.Errorf("scanning upserted item: %w", scanErr) } itemStates[contentID] = true if isNew { created++ } else { updated++ } } queryRows.Close() if queryErr = queryRows.Err(); queryErr != nil { return nil, 0, 0, 0, fmt.Errorf("iterating upserted items: %w", queryErr) } } if onBatch != nil { onBatch(end - start) } } return itemStates, created, updated, skipped, nil } func importItem(ctx context.Context, tx pgx.Tx, item ItemRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) { if mode == ConflictModeSkipExisting { tag, execErr := tx.Exec(ctx, ` INSERT INTO media_items ( content_id, type, title, sort_title, original_title, year, genres, content_rating, runtime, overview, tagline, rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience, imdb_id, tmdb_id, tvdb_id, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, metadata_s3_path, metadata_etag, season_count, studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date, matched_at, last_refreshed, refresh_failures, locked_fields, status, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41 ) ON CONFLICT (content_id) DO NOTHING`, item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres, item.ContentRating, item.Runtime, item.Overview, item.Tagline, item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience, item.ImdbID, item.TmdbID, item.TvdbID, item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath, item.MetadataS3Path, item.MetadataEtag, item.SeasonCount, item.Studios, item.Networks, item.Countries, item.Keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate, item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status, item.CreatedAt, item.UpdatedAt, ) if execErr != nil { return false, false, false, fmt.Errorf("importing item %s: %w", item.ContentID, execErr) } return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil } err = tx.QueryRow(ctx, ` INSERT INTO media_items ( content_id, type, title, sort_title, original_title, year, genres, content_rating, runtime, overview, tagline, rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience, imdb_id, tmdb_id, tvdb_id, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, metadata_s3_path, metadata_etag, season_count, studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date, matched_at, last_refreshed, refresh_failures, locked_fields, status, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41 ) ON CONFLICT (content_id) DO UPDATE SET type = EXCLUDED.type, title = EXCLUDED.title, sort_title = EXCLUDED.sort_title, original_title = EXCLUDED.original_title, year = EXCLUDED.year, genres = EXCLUDED.genres, content_rating = EXCLUDED.content_rating, runtime = EXCLUDED.runtime, overview = EXCLUDED.overview, tagline = EXCLUDED.tagline, rating_imdb = EXCLUDED.rating_imdb, rating_tmdb = EXCLUDED.rating_tmdb, rating_rt_critic = EXCLUDED.rating_rt_critic, rating_rt_audience = EXCLUDED.rating_rt_audience, imdb_id = EXCLUDED.imdb_id, tmdb_id = EXCLUDED.tmdb_id, tvdb_id = EXCLUDED.tvdb_id, poster_path = EXCLUDED.poster_path, poster_thumbhash = EXCLUDED.poster_thumbhash, backdrop_path = EXCLUDED.backdrop_path, backdrop_thumbhash = EXCLUDED.backdrop_thumbhash, logo_path = EXCLUDED.logo_path, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, season_count = EXCLUDED.season_count, studios = EXCLUDED.studios, networks = EXCLUDED.networks, countries = EXCLUDED.countries, keywords = EXCLUDED.keywords, original_language = EXCLUDED.original_language, release_date = EXCLUDED.release_date, first_air_date = EXCLUDED.first_air_date, last_air_date = EXCLUDED.last_air_date, matched_at = EXCLUDED.matched_at, last_refreshed = EXCLUDED.last_refreshed, refresh_failures = EXCLUDED.refresh_failures, locked_fields = EXCLUDED.locked_fields, status = EXCLUDED.status, updated_at = EXCLUDED.updated_at RETURNING (xmax = 0)`, item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres, item.ContentRating, item.Runtime, item.Overview, item.Tagline, item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience, item.ImdbID, item.TmdbID, item.TvdbID, item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath, item.MetadataS3Path, item.MetadataEtag, item.SeasonCount, item.Studios, item.Networks, item.Countries, item.Keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate, item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status, item.CreatedAt, item.UpdatedAt, ).Scan(&created) if err != nil { return false, false, false, fmt.Errorf("upserting item %s: %w", item.ContentID, err) } return created, !created, false, nil } func importSeason(ctx context.Context, tx pgx.Tx, season SeasonRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) { if mode == ConflictModeSkipExisting { tag, execErr := tx.Exec(ctx, ` INSERT INTO seasons ( content_id, series_id, season_number, title, overview, air_date, poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (content_id) DO NOTHING`, season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate, season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag, season.CreatedAt, season.UpdatedAt, ) if execErr != nil { return false, false, false, fmt.Errorf("importing season %s: %w", season.ContentID, execErr) } return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil } err = tx.QueryRow(ctx, ` INSERT INTO seasons ( content_id, series_id, season_number, title, overview, air_date, poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (content_id) DO UPDATE SET series_id = EXCLUDED.series_id, season_number = EXCLUDED.season_number, title = EXCLUDED.title, overview = EXCLUDED.overview, air_date = EXCLUDED.air_date, poster_path = EXCLUDED.poster_path, poster_thumbhash = EXCLUDED.poster_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, updated_at = EXCLUDED.updated_at RETURNING (xmax = 0)`, season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate, season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag, season.CreatedAt, season.UpdatedAt, ).Scan(&created) if err != nil { return false, false, false, fmt.Errorf("upserting season %s: %w", season.ContentID, err) } return created, !created, false, nil } func importEpisode(ctx context.Context, tx pgx.Tx, episode EpisodeRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) { seasonID := nullableString(episode.SeasonID) if mode == ConflictModeSkipExisting { tag, execErr := tx.Exec(ctx, ` INSERT INTO episodes ( content_id, series_id, season_id, season_number, episode_number, title, overview, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 ) ON CONFLICT (content_id) DO NOTHING`, episode.ContentID, episode.SeriesID, seasonID, episode.SeasonNumber, episode.EpisodeNumber, episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB, episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag, episode.CreatedAt, episode.UpdatedAt, ) if execErr != nil { return false, false, false, fmt.Errorf("importing episode %s: %w", episode.ContentID, execErr) } return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil } err = tx.QueryRow(ctx, ` INSERT INTO episodes ( content_id, series_id, season_id, season_number, episode_number, title, overview, air_date, runtime, rating_imdb, rating_tmdb, still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 ) ON CONFLICT (content_id) DO UPDATE SET series_id = EXCLUDED.series_id, season_id = EXCLUDED.season_id, season_number = EXCLUDED.season_number, episode_number = EXCLUDED.episode_number, title = EXCLUDED.title, overview = EXCLUDED.overview, air_date = EXCLUDED.air_date, runtime = EXCLUDED.runtime, rating_imdb = EXCLUDED.rating_imdb, rating_tmdb = EXCLUDED.rating_tmdb, still_path = EXCLUDED.still_path, still_thumbhash = EXCLUDED.still_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, updated_at = EXCLUDED.updated_at RETURNING (xmax = 0)`, episode.ContentID, episode.SeriesID, seasonID, episode.SeasonNumber, episode.EpisodeNumber, episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB, episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag, episode.CreatedAt, episode.UpdatedAt, ).Scan(&created) if err != nil { return false, false, false, fmt.Errorf("upserting episode %s: %w", episode.ContentID, err) } return created, !created, false, nil } func importFile(ctx context.Context, tx pgx.Tx, file FileRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) { videoTracksJSON, err := json.Marshal(file.VideoTracks) if err != nil { return false, false, false, fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, err) } audioTracksJSON, err := json.Marshal(file.AudioTracks) if err != nil { return false, false, false, fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, err) } subtitleTracksJSON, err := json.Marshal(file.SubtitleTracks) if err != nil { return false, false, false, fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, err) } externalSubtitlesJSON, err := json.Marshal(file.ExternalSubtitles) if err != nil { return false, false, false, fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, err) } contentID := nullableString(file.ContentID) episodeID := nullableString(file.EpisodeID) fileHash := nullableString(file.FileHash) probeSource := nullableString(file.ProbeSource) if mode == ConflictModeSkipExisting { tag, execErr := tx.Exec(ctx, ` INSERT INTO media_files ( content_id, episode_id, season_number, episode_number, media_folder_id, file_path, file_size, file_hash, codec_video, codec_audio, resolution, audio_channels, hdr, container, duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles, intro_start, intro_end, credits_start, credits_end, probe_source, probe_updated_at, missing_since, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29 ) ON CONFLICT (file_path) DO NOTHING`, contentID, episodeID, nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber), file.MediaFolderID, file.FilePath, file.FileSize, fileHash, file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container, nilIfZero(file.Duration), nilIfZero(file.Bitrate), videoTracksJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON, file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd, probeSource, file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt, ) if execErr != nil { return false, false, false, fmt.Errorf("importing file %s: %w", file.FilePath, execErr) } return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil } err = tx.QueryRow(ctx, ` INSERT INTO media_files ( content_id, episode_id, season_number, episode_number, media_folder_id, file_path, file_size, file_hash, codec_video, codec_audio, resolution, audio_channels, hdr, container, duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles, intro_start, intro_end, credits_start, credits_end, probe_source, probe_updated_at, missing_since, created_at, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29 ) ON CONFLICT (file_path) DO UPDATE SET content_id = EXCLUDED.content_id, episode_id = EXCLUDED.episode_id, season_number = EXCLUDED.season_number, episode_number = EXCLUDED.episode_number, media_folder_id = EXCLUDED.media_folder_id, file_size = EXCLUDED.file_size, file_hash = EXCLUDED.file_hash, codec_video = EXCLUDED.codec_video, codec_audio = EXCLUDED.codec_audio, resolution = EXCLUDED.resolution, audio_channels = EXCLUDED.audio_channels, hdr = EXCLUDED.hdr, container = EXCLUDED.container, duration = EXCLUDED.duration, bitrate = EXCLUDED.bitrate, video_tracks = EXCLUDED.video_tracks, audio_tracks = EXCLUDED.audio_tracks, subtitle_tracks = EXCLUDED.subtitle_tracks, external_subtitles = EXCLUDED.external_subtitles, intro_start = EXCLUDED.intro_start, intro_end = EXCLUDED.intro_end, credits_start = EXCLUDED.credits_start, credits_end = EXCLUDED.credits_end, probe_source = EXCLUDED.probe_source, probe_updated_at = EXCLUDED.probe_updated_at, missing_since = EXCLUDED.missing_since, updated_at = EXCLUDED.updated_at RETURNING (xmax = 0)`, contentID, episodeID, nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber), file.MediaFolderID, file.FilePath, file.FileSize, fileHash, file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container, nilIfZero(file.Duration), nilIfZero(file.Bitrate), videoTracksJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON, file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd, probeSource, file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt, ).Scan(&created) if err != nil { return false, false, false, fmt.Errorf("upserting file %s: %w", file.FilePath, err) } return created, !created, false, nil } func importLibraryLink(ctx context.Context, tx pgx.Tx, link LibraryLinkRecord) (bool, error) { tag, err := tx.Exec(ctx, ` INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at) VALUES ($1, $2, $3) ON CONFLICT (content_id, media_folder_id) DO NOTHING`, link.ContentID, link.MediaFolderID, link.FirstSeenAt, ) if err != nil { return false, fmt.Errorf("importing library link %s:%d: %w", link.ContentID, link.MediaFolderID, err) } return tag.RowsAffected() == 1, nil } func normalizeRewrites(rewrites []PathRewrite) []PathRewrite { out := make([]PathRewrite, 0, len(rewrites)) for _, rewrite := range rewrites { from := normalizePath(rewrite.From) to := normalizePath(rewrite.To) if from == "" || to == "" { continue } out = append(out, PathRewrite{From: from, To: to}) } sort.Slice(out, func(i, j int) bool { return len(out[i].From) > len(out[j].From) }) return out } func rewritePath(path string, rewrites []PathRewrite) (string, bool) { normalized := normalizePath(path) for _, rewrite := range rewrites { if !hasPathPrefix(normalized, rewrite.From) { continue } if normalized == rewrite.From { return rewrite.To, true } suffix := strings.TrimPrefix(normalized, rewrite.From) return normalizePath(rewrite.To + suffix), true } return normalized, false } func normalizePath(path string) string { if path == "" { return "" } return filepath.Clean(path) } func hasPathPrefix(path, prefix string) bool { path = normalizePath(path) prefix = normalizePath(prefix) if path == prefix { return true } if prefix == string(os.PathSeparator) { return strings.HasPrefix(path, prefix) } return strings.HasPrefix(path, prefix+string(os.PathSeparator)) } func sortLibraries(records []LibraryRecord) { sort.Slice(records, func(i, j int) bool { return records[i].ExportedID < records[j].ExportedID }) } func cloneStrings(values []string) []string { if values == nil { return []string{} } return append([]string(nil), values...) } func itemRecordStringArrays(item ItemRecord) ([]string, []string, []string, []string) { return nonNilStrings(item.Studios), nonNilStrings(item.Networks), nonNilStrings(item.Countries), nonNilStrings(item.Keywords) } func nonNilStrings(values []string) []string { if values == nil { return []string{} } return values } func cloneInts(values []int) []int { if values == nil { return []int{} } return append([]int(nil), values...) } func dedupeStrings(values []string) []string { if len(values) == 0 { return []string{} } seen := make(map[string]struct{}, len(values)) out := make([]string, 0, len(values)) for _, value := range values { if _, ok := seen[value]; ok { continue } seen[value] = struct{}{} out = append(out, value) } return out } func mapKeys(m map[string]struct{}) []string { keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) } return keys } func nilIfZero(value int) any { if value == 0 { return nil } return value } func nullableString(value string) any { if value == "" { return nil } return value } func toVideoTrackRecords(tracks []models.VideoTrack) []VideoTrackRecord { out := make([]VideoTrackRecord, 0, len(tracks)) for _, track := range tracks { out = append(out, VideoTrackRecord{ Title: track.Title, Codec: track.Codec, DolbyVision: track.DolbyVision, Profile: track.Profile, Level: track.Level, Width: track.Width, Height: track.Height, AspectRatio: track.AspectRatio, Interlaced: track.Interlaced, FrameRate: track.FrameRate, Bitrate: track.Bitrate, VideoRange: track.VideoRange, ColorPrimaries: track.ColorPrimaries, ColorSpace: track.ColorSpace, ColorTransfer: track.ColorTransfer, BitDepth: track.BitDepth, PixelFormat: track.PixelFormat, ReferenceFrames: track.ReferenceFrames, }) } return out } func toAudioTrackRecords(tracks []models.AudioTrack) []AudioTrackRecord { out := make([]AudioTrackRecord, 0, len(tracks)) for _, track := range tracks { out = append(out, AudioTrackRecord{ Title: track.Title, EmbeddedTitle: track.EmbeddedTitle, Language: track.Language, Codec: track.Codec, Layout: track.Layout, Channels: track.Channels, Bitrate: track.Bitrate, SampleRate: track.SampleRate, BitDepth: track.BitDepth, Default: track.Default, }) } return out } func toSubtitleTrackRecords(tracks []models.SubtitleTrack) []SubtitleTrackRecord { out := make([]SubtitleTrackRecord, 0, len(tracks)) for _, track := range tracks { out = append(out, SubtitleTrackRecord{ Index: track.Index, Language: track.Language, Codec: track.Codec, Title: track.Title, EmbeddedTitle: track.EmbeddedTitle, Resolution: track.Resolution, Forced: track.Forced, Default: track.Default, HearingImpaired: track.HearingImpaired, External: track.External, FileName: track.FileName, }) } return out } func toExternalSubtitleRecords(tracks []models.ExternalSubtitle) []ExternalSubtitleRecord { out := make([]ExternalSubtitleRecord, 0, len(tracks)) for _, track := range tracks { out = append(out, ExternalSubtitleRecord{ Path: track.Path, Language: track.Language, Format: track.Format, Title: track.Title, EmbeddedTitle: track.EmbeddedTitle, Resolution: track.Resolution, Forced: track.Forced, Default: track.Default, HearingImpaired: track.HearingImpaired, }) } return out }