2601 lines
87 KiB
Go
2601 lines
87 KiB
Go
package catalogseed
|
||
|
||
import (
|
||
"bytes"
|
||
"compress/gzip"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
pgvector "github.com/pgvector/pgvector-go"
|
||
|
||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||
"github.com/Silo-Server/silo-server/internal/idgen"
|
||
"github.com/Silo-Server/silo-server/internal/lang"
|
||
"github.com/Silo-Server/silo-server/internal/models"
|
||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||
"github.com/Silo-Server/silo-server/internal/titleutil"
|
||
)
|
||
|
||
var (
|
||
ErrUnsupportedBundleVersion = errors.New("unsupported catalog seed version")
|
||
ErrInvalidConflictMode = errors.New("invalid catalog seed conflict mode")
|
||
ErrInvalidBundle = errors.New("invalid catalog seed bundle")
|
||
)
|
||
|
||
type UnmatchedRootsError struct {
|
||
Roots []string
|
||
}
|
||
|
||
func (e *UnmatchedRootsError) Error() string {
|
||
if len(e.Roots) == 0 {
|
||
return "catalog seed import requires additional path rewrites"
|
||
}
|
||
return fmt.Sprintf("catalog seed import requires additional path rewrites: %s", strings.Join(e.Roots, ", "))
|
||
}
|
||
|
||
type Service struct {
|
||
pool *pgxpool.Pool
|
||
personRepo *catalog.PersonRepository
|
||
recsRepo *recommendations.Repo
|
||
}
|
||
|
||
func NewService(pool *pgxpool.Pool, personRepo *catalog.PersonRepository, recsRepo *recommendations.Repo) *Service {
|
||
if personRepo == nil {
|
||
personRepo = catalog.NewPersonRepository(pool)
|
||
}
|
||
if recsRepo == nil {
|
||
recsRepo = recommendations.NewRepo(pool)
|
||
}
|
||
return &Service{pool: pool, personRepo: personRepo, recsRepo: recsRepo}
|
||
}
|
||
|
||
func (s *Service) Export(ctx context.Context, opts ExportOptions) ([]byte, error) {
|
||
var raw bytes.Buffer
|
||
if _, err := s.ExportToWriter(ctx, &raw, opts, nil); err != nil {
|
||
return nil, err
|
||
}
|
||
return raw.Bytes(), nil
|
||
}
|
||
|
||
func (s *Service) Import(ctx context.Context, data []byte, opts ImportOptions) (*ImportResult, error) {
|
||
return s.ImportWithProgress(ctx, data, opts, nil)
|
||
}
|
||
|
||
func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts ImportOptions, progress func(ImportProgress)) (*ImportResult, error) {
|
||
if opts.ConflictMode == "" {
|
||
opts.ConflictMode = ConflictModeSkipExisting
|
||
}
|
||
if opts.ConflictMode != ConflictModeSkipExisting && opts.ConflictMode != ConflictModeOverwrite {
|
||
return nil, ErrInvalidConflictMode
|
||
}
|
||
|
||
reportProgress := func(message string, current, total int) {
|
||
if progress == nil {
|
||
return
|
||
}
|
||
progress(ImportProgress{
|
||
Message: message,
|
||
Current: current,
|
||
Total: total,
|
||
})
|
||
}
|
||
|
||
reportProgress("Reading catalog bundle", 0, 0)
|
||
bundle, err := decodeBundle(data)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if bundle.Manifest.FormatVersion != CurrentBundleVersion {
|
||
return nil, ErrUnsupportedBundleVersion
|
||
}
|
||
for i := range bundle.Items {
|
||
bundle.Items[i].OriginalLanguage = lang.Canonical(bundle.Items[i].OriginalLanguage)
|
||
bundle.Items[i].Countries = lang.CanonicalCountries(bundle.Items[i].Countries)
|
||
}
|
||
|
||
totalWork := len(bundle.Libraries) + len(bundle.Items) + len(bundle.Embeddings) + len(bundle.Seasons) + len(bundle.Episodes) + len(bundle.Files) + len(bundle.LibraryLinks) + 1
|
||
if totalWork <= 0 {
|
||
totalWork = 1
|
||
}
|
||
currentWork := 0
|
||
reportProgress("Validating catalog bundle", currentWork, totalWork)
|
||
|
||
rewrites := normalizeRewrites(opts.PathRewrites)
|
||
libraries, folderPaths, unmatchedRoots, err := prepareLibraries(bundle.Libraries, rewrites)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if len(unmatchedRoots) > 0 {
|
||
sort.Strings(unmatchedRoots)
|
||
return nil, &UnmatchedRootsError{Roots: unmatchedRoots}
|
||
}
|
||
|
||
if err := validateFileRoots(bundle.Files, folderPaths, rewrites); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
reportProgress("Starting catalog import", currentWork, totalWork)
|
||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||
if err != nil {
|
||
return nil, fmt.Errorf("beginning catalog seed import: %w", err)
|
||
}
|
||
defer func() { _ = tx.Rollback(ctx) }()
|
||
|
||
emptyTarget, err := isEmptyCatalogImportTarget(ctx, tx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
result := &ImportResult{}
|
||
folderIDMap := make(map[int]int, len(libraries))
|
||
for _, library := range libraries {
|
||
localID, created, matched, importErr := importLibrary(ctx, tx, library, opts.ConflictMode == ConflictModeOverwrite)
|
||
if importErr != nil {
|
||
return nil, importErr
|
||
}
|
||
folderIDMap[library.ExportedID] = localID
|
||
if created {
|
||
result.LibrariesCreated++
|
||
}
|
||
if matched {
|
||
result.LibrariesMatched++
|
||
}
|
||
currentWork++
|
||
reportProgress("Importing libraries", currentWork, totalWork)
|
||
}
|
||
|
||
if emptyTarget {
|
||
// Empty-target fast path: use COPY protocol for maximum throughput.
|
||
itemRows := make([][]any, 0, len(bundle.Items))
|
||
for _, item := range bundle.Items {
|
||
studios, networks, countries, keywords := itemRecordStringArrays(item)
|
||
itemRows = append(itemRows, []any{
|
||
item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres,
|
||
item.ContentRating, item.Runtime, item.Overview, item.Tagline,
|
||
item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience,
|
||
item.ImdbID, item.TmdbID, item.TvdbID,
|
||
item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath,
|
||
item.MetadataS3Path, item.MetadataEtag, item.SeasonCount,
|
||
studios, networks, countries, keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate,
|
||
item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status,
|
||
item.CreatedAt, item.UpdatedAt,
|
||
})
|
||
}
|
||
if err := copyInsertBatches(ctx, tx, "media_items",
|
||
[]string{
|
||
"content_id", "type", "title", "sort_title", "original_title", "year", "genres",
|
||
"content_rating", "runtime", "overview", "tagline",
|
||
"rating_imdb", "rating_tmdb", "rating_rt_critic", "rating_rt_audience",
|
||
"imdb_id", "tmdb_id", "tvdb_id",
|
||
"poster_path", "poster_thumbhash", "backdrop_path", "backdrop_thumbhash", "logo_path",
|
||
"metadata_s3_path", "metadata_etag", "season_count",
|
||
"studios", "networks", "countries", "keywords", "original_language", "release_date", "first_air_date", "last_air_date",
|
||
"matched_at", "last_refreshed", "refresh_failures", "locked_fields", "status",
|
||
"created_at", "updated_at",
|
||
},
|
||
itemRows, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing catalog items", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
result.ItemsCreated = len(bundle.Items)
|
||
|
||
// Build lookup sets for FK filtering of child records.
|
||
itemIDs := make(map[string]struct{}, len(bundle.Items))
|
||
itemStates := make(map[string]bool, len(bundle.Items))
|
||
for _, item := range bundle.Items {
|
||
itemIDs[item.ContentID] = struct{}{}
|
||
itemStates[item.ContentID] = true
|
||
}
|
||
|
||
reportProgress("Importing people", currentWork, totalWork)
|
||
if err := s.replacePeople(ctx, tx, bundle.People, itemStates, ConflictModeOverwrite, result); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := s.importEmbeddings(ctx, tx, bundle.Embeddings, true, result, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing embeddings", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
currentWork++
|
||
reportProgress("Importing embeddings", currentWork, totalWork)
|
||
|
||
seasonRows := make([][]any, 0, len(bundle.Seasons))
|
||
seasonIDs := make(map[string]struct{}, len(bundle.Seasons))
|
||
for _, season := range bundle.Seasons {
|
||
if _, ok := itemIDs[season.SeriesID]; !ok {
|
||
continue // skip orphan season
|
||
}
|
||
seasonIDs[season.ContentID] = struct{}{}
|
||
seasonRows = append(seasonRows, []any{
|
||
season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate,
|
||
season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag,
|
||
season.CreatedAt, season.UpdatedAt,
|
||
})
|
||
}
|
||
if err := copyInsertBatches(ctx, tx, "seasons",
|
||
[]string{
|
||
"content_id", "series_id", "season_number", "title", "overview", "air_date",
|
||
"poster_path", "poster_thumbhash", "metadata_s3_path", "metadata_etag", "created_at", "updated_at",
|
||
},
|
||
seasonRows, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing seasons", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
result.SeasonsCreated = len(seasonRows)
|
||
|
||
episodeRows := make([][]any, 0, len(bundle.Episodes))
|
||
episodeIDs := make(map[string]struct{}, len(bundle.Episodes))
|
||
for _, episode := range bundle.Episodes {
|
||
// Skip episodes whose series or season doesn't exist.
|
||
if _, ok := itemIDs[episode.SeriesID]; !ok {
|
||
continue
|
||
}
|
||
if episode.SeasonID != "" {
|
||
if _, ok := seasonIDs[episode.SeasonID]; !ok {
|
||
continue
|
||
}
|
||
}
|
||
episodeIDs[episode.ContentID] = struct{}{}
|
||
episodeRows = append(episodeRows, []any{
|
||
episode.ContentID, episode.SeriesID, nullableString(episode.SeasonID), episode.SeasonNumber, episode.EpisodeNumber,
|
||
episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB,
|
||
episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag,
|
||
episode.CreatedAt, episode.UpdatedAt,
|
||
})
|
||
}
|
||
if err := copyInsertBatches(ctx, tx, "episodes",
|
||
[]string{
|
||
"content_id", "series_id", "season_id", "season_number", "episode_number",
|
||
"title", "overview", "air_date", "runtime", "rating_imdb", "rating_tmdb",
|
||
"still_path", "still_thumbhash", "metadata_s3_path", "metadata_etag", "created_at", "updated_at",
|
||
},
|
||
episodeRows, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing episodes", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
result.EpisodesCreated = len(episodeRows)
|
||
|
||
rewrittenFiles := make([]FileRecord, 0, len(bundle.Files))
|
||
for _, file := range bundle.Files {
|
||
// Skip files referencing non-existent items or episodes.
|
||
if file.ContentID != "" {
|
||
if _, ok := itemIDs[file.ContentID]; !ok {
|
||
continue
|
||
}
|
||
}
|
||
if file.EpisodeID != "" {
|
||
if _, ok := episodeIDs[file.EpisodeID]; !ok {
|
||
continue
|
||
}
|
||
}
|
||
rewritten, rewriteErr := rewriteFileRecord(file, rewrites, folderPaths)
|
||
if rewriteErr != nil {
|
||
return nil, rewriteErr
|
||
}
|
||
rewritten.MediaFolderID = folderIDMap[file.MediaFolderID]
|
||
rewrittenFiles = append(rewrittenFiles, rewritten)
|
||
}
|
||
fileRows := make([][]any, 0, len(rewrittenFiles))
|
||
for _, file := range rewrittenFiles {
|
||
videoTracksJSON, jsonErr := json.Marshal(file.VideoTracks)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
audioTracksJSON, jsonErr := json.Marshal(file.AudioTracks)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
subtitleTracksJSON, jsonErr := json.Marshal(file.SubtitleTracks)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
externalSubtitlesJSON, jsonErr := json.Marshal(file.ExternalSubtitles)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
fileRows = append(fileRows, []any{
|
||
nullableString(file.ContentID), nullableString(file.EpisodeID), nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber),
|
||
file.MediaFolderID, file.FilePath, file.FileSize, nullableString(file.FileHash),
|
||
file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container,
|
||
nilIfZero(file.Duration), nilIfZero(file.Bitrate), videoTracksJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON,
|
||
file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd,
|
||
nullableString(file.ProbeSource), file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt,
|
||
})
|
||
}
|
||
if err := copyInsertBatches(ctx, tx, "media_files",
|
||
[]string{
|
||
"content_id", "episode_id", "season_number", "episode_number",
|
||
"media_folder_id", "file_path", "file_size", "file_hash",
|
||
"codec_video", "codec_audio", "resolution", "audio_channels", "hdr", "container",
|
||
"duration", "bitrate", "video_tracks", "audio_tracks", "subtitle_tracks", "external_subtitles",
|
||
"intro_start", "intro_end", "credits_start", "credits_end",
|
||
"probe_source", "probe_updated_at", "missing_since", "created_at", "updated_at",
|
||
},
|
||
fileRows, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing media files", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
result.FilesCreated = len(rewrittenFiles)
|
||
|
||
linkRows := make([][]any, 0, len(bundle.LibraryLinks))
|
||
for _, link := range bundle.LibraryLinks {
|
||
if _, ok := itemIDs[link.ContentID]; !ok {
|
||
continue
|
||
}
|
||
linkRows = append(linkRows, []any{
|
||
link.ContentID, folderIDMap[link.MediaFolderID], link.FirstSeenAt,
|
||
})
|
||
}
|
||
if err := copyInsertBatches(ctx, tx, "media_item_libraries",
|
||
[]string{"content_id", "media_folder_id", "first_seen_at"},
|
||
linkRows, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Linking items to libraries", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, err
|
||
}
|
||
result.LinksCreated = len(linkRows)
|
||
|
||
reportProgress("Finalizing catalog import", currentWork, totalWork)
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return nil, fmt.Errorf("committing catalog seed import: %w", err)
|
||
}
|
||
|
||
reportProgress("Catalog import completed", totalWork, totalWork)
|
||
return result, nil
|
||
}
|
||
|
||
// Batch import items (instead of one-by-one).
|
||
itemStates, itemsCreated, itemsUpdated, itemsSkipped, err := batchImportItems(ctx, tx, bundle.Items, opts.ConflictMode, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing catalog items", currentWork, totalWork)
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result.ItemsCreated = itemsCreated
|
||
result.ItemsUpdated = itemsUpdated
|
||
result.Skipped += itemsSkipped
|
||
|
||
reportProgress("Importing people", currentWork, totalWork)
|
||
if err := s.replacePeople(ctx, tx, bundle.People, itemStates, opts.ConflictMode, result); err != nil {
|
||
return nil, fmt.Errorf("replacing people: %w", err)
|
||
}
|
||
if err := s.importEmbeddings(ctx, tx, bundle.Embeddings, false, result, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing embeddings", currentWork, totalWork)
|
||
}); err != nil {
|
||
return nil, fmt.Errorf("importing embeddings: %w", err)
|
||
}
|
||
currentWork++
|
||
reportProgress("Importing embeddings", currentWork, totalWork)
|
||
|
||
// Batch import seasons.
|
||
seasonRows := make([][]any, 0, len(bundle.Seasons))
|
||
for _, season := range bundle.Seasons {
|
||
seasonRows = append(seasonRows, []any{
|
||
season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate,
|
||
season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag,
|
||
season.CreatedAt, season.UpdatedAt,
|
||
})
|
||
}
|
||
seasonCounts, err := executeUpsertBatches(ctx, tx, `
|
||
INSERT INTO seasons (
|
||
content_id, series_id, season_number, title, overview, air_date,
|
||
poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES `,
|
||
seasonRows, 12, nil,
|
||
` ON CONFLICT (content_id) DO NOTHING`,
|
||
`
|
||
ON CONFLICT (content_id) DO UPDATE SET
|
||
series_id = EXCLUDED.series_id,
|
||
season_number = EXCLUDED.season_number,
|
||
title = EXCLUDED.title,
|
||
overview = EXCLUDED.overview,
|
||
air_date = EXCLUDED.air_date,
|
||
poster_path = EXCLUDED.poster_path,
|
||
poster_thumbhash = EXCLUDED.poster_thumbhash,
|
||
metadata_s3_path = EXCLUDED.metadata_s3_path,
|
||
metadata_etag = EXCLUDED.metadata_etag,
|
||
updated_at = EXCLUDED.updated_at`,
|
||
opts.ConflictMode, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing seasons", currentWork, totalWork)
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result.SeasonsCreated = seasonCounts.Created
|
||
result.SeasonsUpdated = seasonCounts.Updated
|
||
result.Skipped += seasonCounts.Skipped
|
||
|
||
// Batch import episodes.
|
||
episodeRows := make([][]any, 0, len(bundle.Episodes))
|
||
for _, episode := range bundle.Episodes {
|
||
episodeRows = append(episodeRows, []any{
|
||
episode.ContentID, episode.SeriesID, nullableString(episode.SeasonID), episode.SeasonNumber, episode.EpisodeNumber,
|
||
episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB,
|
||
episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag,
|
||
episode.CreatedAt, episode.UpdatedAt,
|
||
})
|
||
}
|
||
episodeCounts, err := executeUpsertBatches(ctx, tx, `
|
||
INSERT INTO episodes (
|
||
content_id, series_id, season_id, season_number, episode_number,
|
||
title, overview, air_date, runtime, rating_imdb, rating_tmdb,
|
||
still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES `,
|
||
episodeRows, 17, nil,
|
||
` ON CONFLICT (content_id) DO NOTHING`,
|
||
`
|
||
ON CONFLICT (content_id) DO UPDATE SET
|
||
series_id = EXCLUDED.series_id,
|
||
season_id = EXCLUDED.season_id,
|
||
season_number = EXCLUDED.season_number,
|
||
episode_number = EXCLUDED.episode_number,
|
||
title = EXCLUDED.title,
|
||
overview = EXCLUDED.overview,
|
||
air_date = EXCLUDED.air_date,
|
||
runtime = EXCLUDED.runtime,
|
||
rating_imdb = EXCLUDED.rating_imdb,
|
||
rating_tmdb = EXCLUDED.rating_tmdb,
|
||
still_path = EXCLUDED.still_path,
|
||
still_thumbhash = EXCLUDED.still_thumbhash,
|
||
metadata_s3_path = EXCLUDED.metadata_s3_path,
|
||
metadata_etag = EXCLUDED.metadata_etag,
|
||
updated_at = EXCLUDED.updated_at`,
|
||
opts.ConflictMode, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing episodes", currentWork, totalWork)
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result.EpisodesCreated = episodeCounts.Created
|
||
result.EpisodesUpdated = episodeCounts.Updated
|
||
result.Skipped += episodeCounts.Skipped
|
||
|
||
// Rewrite file paths and batch import files.
|
||
rewrittenFiles := make([]FileRecord, 0, len(bundle.Files))
|
||
for _, file := range bundle.Files {
|
||
rewritten, rewriteErr := rewriteFileRecord(file, rewrites, folderPaths)
|
||
if rewriteErr != nil {
|
||
return nil, rewriteErr
|
||
}
|
||
rewritten.MediaFolderID = folderIDMap[file.MediaFolderID]
|
||
rewrittenFiles = append(rewrittenFiles, rewritten)
|
||
}
|
||
fileRows := make([][]any, 0, len(rewrittenFiles))
|
||
for _, file := range rewrittenFiles {
|
||
videoTracksJSON, jsonErr := json.Marshal(file.VideoTracks)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
audioTracksJSON, jsonErr := json.Marshal(file.AudioTracks)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
subtitleTracksJSON, jsonErr := json.Marshal(file.SubtitleTracks)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
externalSubtitlesJSON, jsonErr := json.Marshal(file.ExternalSubtitles)
|
||
if jsonErr != nil {
|
||
return nil, fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, jsonErr)
|
||
}
|
||
fileRows = append(fileRows, []any{
|
||
nullableString(file.ContentID), nullableString(file.EpisodeID), nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber),
|
||
file.MediaFolderID, file.FilePath, file.FileSize, nullableString(file.FileHash),
|
||
file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container,
|
||
nilIfZero(file.Duration), nilIfZero(file.Bitrate), string(videoTracksJSON), string(audioTracksJSON), string(subtitleTracksJSON), string(externalSubtitlesJSON),
|
||
file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd,
|
||
nullableString(file.ProbeSource), file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt,
|
||
})
|
||
}
|
||
fileCounts, err := executeUpsertBatches(ctx, tx, `
|
||
INSERT INTO media_files (
|
||
content_id, episode_id, season_number, episode_number,
|
||
media_folder_id, file_path, file_size, file_hash,
|
||
codec_video, codec_audio, resolution, audio_channels, hdr, container,
|
||
duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles,
|
||
intro_start, intro_end, credits_start, credits_end,
|
||
probe_source, probe_updated_at, missing_since, created_at, updated_at
|
||
) VALUES `,
|
||
fileRows, 29,
|
||
map[int]string{16: "::jsonb", 17: "::jsonb", 18: "::jsonb", 19: "::jsonb"},
|
||
` ON CONFLICT (file_path) DO NOTHING`,
|
||
`
|
||
ON CONFLICT (file_path) DO UPDATE SET
|
||
content_id = EXCLUDED.content_id,
|
||
episode_id = EXCLUDED.episode_id,
|
||
season_number = EXCLUDED.season_number,
|
||
episode_number = EXCLUDED.episode_number,
|
||
media_folder_id = EXCLUDED.media_folder_id,
|
||
file_size = EXCLUDED.file_size,
|
||
file_hash = EXCLUDED.file_hash,
|
||
codec_video = EXCLUDED.codec_video,
|
||
codec_audio = EXCLUDED.codec_audio,
|
||
resolution = EXCLUDED.resolution,
|
||
audio_channels = EXCLUDED.audio_channels,
|
||
hdr = EXCLUDED.hdr,
|
||
container = EXCLUDED.container,
|
||
duration = EXCLUDED.duration,
|
||
bitrate = EXCLUDED.bitrate,
|
||
video_tracks = EXCLUDED.video_tracks,
|
||
audio_tracks = EXCLUDED.audio_tracks,
|
||
subtitle_tracks = EXCLUDED.subtitle_tracks,
|
||
external_subtitles = EXCLUDED.external_subtitles,
|
||
intro_start = EXCLUDED.intro_start,
|
||
intro_end = EXCLUDED.intro_end,
|
||
credits_start = EXCLUDED.credits_start,
|
||
credits_end = EXCLUDED.credits_end,
|
||
probe_source = EXCLUDED.probe_source,
|
||
probe_updated_at = EXCLUDED.probe_updated_at,
|
||
missing_since = EXCLUDED.missing_since,
|
||
updated_at = EXCLUDED.updated_at`,
|
||
opts.ConflictMode, func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Importing media files", currentWork, totalWork)
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result.FilesCreated = fileCounts.Created
|
||
result.FilesUpdated = fileCounts.Updated
|
||
result.Skipped += fileCounts.Skipped
|
||
|
||
// Batch import library links.
|
||
linkRows := make([][]any, 0, len(bundle.LibraryLinks))
|
||
for _, link := range bundle.LibraryLinks {
|
||
linkRows = append(linkRows, []any{link.ContentID, folderIDMap[link.MediaFolderID], link.FirstSeenAt})
|
||
}
|
||
linksAffected, err := executeInsertBatchesCounted(ctx, tx, `
|
||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||
VALUES `,
|
||
linkRows, 3, nil,
|
||
` ON CONFLICT (content_id, media_folder_id) DO NOTHING`,
|
||
func(processed int) {
|
||
currentWork += processed
|
||
reportProgress("Linking items to libraries", currentWork, totalWork)
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result.LinksCreated = int(linksAffected)
|
||
|
||
reportProgress("Finalizing catalog import", currentWork, totalWork)
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return nil, fmt.Errorf("committing catalog seed import: %w", err)
|
||
}
|
||
|
||
reportProgress("Catalog import completed", totalWork, totalWork)
|
||
return result, nil
|
||
}
|
||
|
||
func (s *Service) exportFolders(ctx context.Context, libraryIDs []int) ([]LibraryRecord, error) {
|
||
folderRepo := catalog.NewFolderRepository(s.pool)
|
||
var folders []*models.MediaFolder
|
||
if len(libraryIDs) == 0 {
|
||
var err error
|
||
folders, err = folderRepo.List(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("listing folders for export: %w", err)
|
||
}
|
||
} else {
|
||
folders = make([]*models.MediaFolder, 0, len(libraryIDs))
|
||
for _, libraryID := range libraryIDs {
|
||
folder, err := folderRepo.GetByID(ctx, libraryID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("loading folder %d for export: %w", libraryID, err)
|
||
}
|
||
folders = append(folders, folder)
|
||
}
|
||
}
|
||
|
||
records := make([]LibraryRecord, 0, len(folders))
|
||
for _, folder := range folders {
|
||
paths := append([]string(nil), folder.Paths...)
|
||
sort.Strings(paths)
|
||
records = append(records, LibraryRecord{
|
||
ExportedID: folder.ID,
|
||
Paths: paths,
|
||
Type: folder.Type,
|
||
Name: folder.Name,
|
||
Enabled: folder.Enabled,
|
||
LastScannedAt: folder.LastScannedAt,
|
||
ScanWarningCode: folder.ScanWarningCode,
|
||
ScanWarningMessage: folder.ScanWarningMessage,
|
||
ScanWarningAt: folder.ScanWarningAt,
|
||
AllowEmptyCleanupOnce: folder.AllowEmptyCleanupOnce,
|
||
})
|
||
}
|
||
|
||
return records, nil
|
||
}
|
||
|
||
func (s *Service) schemaVersion(ctx context.Context) (int, error) {
|
||
var version int
|
||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_versions").Scan(&version); err != nil {
|
||
return 0, fmt.Errorf("loading schema version: %w", err)
|
||
}
|
||
return version, nil
|
||
}
|
||
|
||
func (s *Service) loadSeasons(ctx context.Context, seriesID string) ([]SeasonRecord, error) {
|
||
rows, err := s.pool.Query(ctx, `
|
||
SELECT content_id, series_id, season_number, title, overview,
|
||
air_date, poster_path, poster_thumbhash, metadata_s3_path, metadata_etag,
|
||
created_at, updated_at
|
||
FROM seasons
|
||
WHERE series_id = $1
|
||
ORDER BY season_number ASC`, seriesID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("querying seasons for %s: %w", seriesID, err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
var seasons []SeasonRecord
|
||
for rows.Next() {
|
||
var record SeasonRecord
|
||
if err := rows.Scan(
|
||
&record.ContentID,
|
||
&record.SeriesID,
|
||
&record.SeasonNumber,
|
||
&record.Title,
|
||
&record.Overview,
|
||
&record.AirDate,
|
||
&record.PosterPath,
|
||
&record.PosterThumbhash,
|
||
&record.MetadataS3Path,
|
||
&record.MetadataEtag,
|
||
&record.CreatedAt,
|
||
&record.UpdatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scanning season for %s: %w", seriesID, err)
|
||
}
|
||
seasons = append(seasons, record)
|
||
}
|
||
return seasons, rows.Err()
|
||
}
|
||
|
||
func (s *Service) loadEpisodes(ctx context.Context, seriesID string) ([]EpisodeRecord, error) {
|
||
rows, err := s.pool.Query(ctx, `
|
||
SELECT content_id, series_id, season_id, season_number, episode_number,
|
||
title, overview, air_date, runtime, rating_imdb, rating_tmdb,
|
||
still_path, still_thumbhash, metadata_s3_path, metadata_etag,
|
||
created_at, updated_at
|
||
FROM episodes
|
||
WHERE series_id = $1
|
||
ORDER BY season_number ASC, episode_number ASC`, seriesID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("querying episodes for %s: %w", seriesID, err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
var episodes []EpisodeRecord
|
||
for rows.Next() {
|
||
var record EpisodeRecord
|
||
var seasonID *string
|
||
if err := rows.Scan(
|
||
&record.ContentID,
|
||
&record.SeriesID,
|
||
&seasonID,
|
||
&record.SeasonNumber,
|
||
&record.EpisodeNumber,
|
||
&record.Title,
|
||
&record.Overview,
|
||
&record.AirDate,
|
||
&record.Runtime,
|
||
&record.RatingIMDB,
|
||
&record.RatingTMDB,
|
||
&record.StillPath,
|
||
&record.StillThumbhash,
|
||
&record.MetadataS3Path,
|
||
&record.MetadataEtag,
|
||
&record.CreatedAt,
|
||
&record.UpdatedAt,
|
||
); err != nil {
|
||
return nil, fmt.Errorf("scanning episode for %s: %w", seriesID, err)
|
||
}
|
||
if seasonID != nil {
|
||
record.SeasonID = *seasonID
|
||
}
|
||
episodes = append(episodes, record)
|
||
}
|
||
return episodes, rows.Err()
|
||
}
|
||
|
||
func itemToRecord(item *models.MediaItem) ItemRecord {
|
||
if strings.TrimSpace(item.SortTitle) == "" {
|
||
if derived := titleutil.DeriveDefaultSortTitle(item.Title); derived != "" {
|
||
item.SortTitle = derived
|
||
}
|
||
}
|
||
return ItemRecord{
|
||
ContentID: item.ContentID,
|
||
Type: item.Type,
|
||
Title: item.Title,
|
||
SortTitle: item.SortTitle,
|
||
OriginalTitle: item.OriginalTitle,
|
||
Year: item.Year,
|
||
Genres: cloneStrings(item.Genres),
|
||
ContentRating: item.ContentRating,
|
||
Runtime: item.Runtime,
|
||
Overview: item.Overview,
|
||
Tagline: item.Tagline,
|
||
RatingIMDB: item.RatingIMDB,
|
||
RatingTMDB: item.RatingTMDB,
|
||
RatingRTCritic: item.RatingRTCritic,
|
||
RatingRTAudience: item.RatingRTAudience,
|
||
ImdbID: item.ImdbID,
|
||
TmdbID: item.TmdbID,
|
||
TvdbID: item.TvdbID,
|
||
PosterPath: item.PosterPath,
|
||
PosterThumbhash: item.PosterThumbhash,
|
||
BackdropPath: item.BackdropPath,
|
||
BackdropThumbhash: item.BackdropThumbhash,
|
||
LogoPath: item.LogoPath,
|
||
MetadataS3Path: item.MetadataS3Path,
|
||
MetadataEtag: item.MetadataEtag,
|
||
SeasonCount: item.SeasonCount,
|
||
Studios: cloneStrings(item.Studios),
|
||
Networks: cloneStrings(item.Networks),
|
||
Countries: cloneStrings(item.Countries),
|
||
ReleaseDate: item.ReleaseDate,
|
||
FirstAirDate: item.FirstAirDate,
|
||
LastAirDate: item.LastAirDate,
|
||
MatchedAt: item.MatchedAt,
|
||
LastRefreshed: item.LastRefreshed,
|
||
RefreshFailures: item.RefreshFailures,
|
||
LockedFields: cloneInts(item.LockedFields),
|
||
Status: item.Status,
|
||
CreatedAt: item.CreatedAt,
|
||
UpdatedAt: item.UpdatedAt,
|
||
}
|
||
}
|
||
|
||
func fileToRecord(file *models.MediaFile) FileRecord {
|
||
return FileRecord{
|
||
ContentID: file.ContentID,
|
||
EpisodeID: file.EpisodeID,
|
||
SeasonNumber: file.SeasonNumber,
|
||
EpisodeNumber: file.EpisodeNumber,
|
||
MediaFolderID: file.MediaFolderID,
|
||
FilePath: file.FilePath,
|
||
FileSize: file.FileSize,
|
||
FileHash: file.FileHash,
|
||
CodecVideo: file.CodecVideo,
|
||
CodecAudio: file.CodecAudio,
|
||
Resolution: file.Resolution,
|
||
AudioChannels: file.AudioChannels,
|
||
HDR: file.HDR,
|
||
Container: file.Container,
|
||
Duration: file.Duration,
|
||
Bitrate: file.Bitrate,
|
||
VideoTracks: toVideoTrackRecords(file.VideoTracks),
|
||
AudioTracks: toAudioTrackRecords(file.AudioTracks),
|
||
SubtitleTracks: toSubtitleTrackRecords(file.SubtitleTracks),
|
||
ExternalSubtitles: toExternalSubtitleRecords(file.ExternalSubtitles),
|
||
IntroStart: file.IntroStart,
|
||
IntroEnd: file.IntroEnd,
|
||
CreditsStart: file.CreditsStart,
|
||
CreditsEnd: file.CreditsEnd,
|
||
ProbeSource: file.ProbeSource,
|
||
ProbeUpdatedAt: file.ProbeUpdatedAt,
|
||
MissingSince: file.MissingSince,
|
||
CreatedAt: file.CreatedAt,
|
||
UpdatedAt: file.UpdatedAt,
|
||
}
|
||
}
|
||
|
||
func encodeBundle(bundle Bundle) ([]byte, error) {
|
||
var raw bytes.Buffer
|
||
gz := gzip.NewWriter(&raw)
|
||
if err := json.NewEncoder(gz).Encode(bundle); err != nil {
|
||
_ = gz.Close()
|
||
return nil, fmt.Errorf("encoding catalog seed bundle: %w", err)
|
||
}
|
||
if err := gz.Close(); err != nil {
|
||
return nil, fmt.Errorf("closing catalog seed bundle: %w", err)
|
||
}
|
||
return raw.Bytes(), nil
|
||
}
|
||
|
||
func decodeBundle(data []byte) (*Bundle, error) {
|
||
gz, err := gzip.NewReader(bytes.NewReader(data))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("%w: opening catalog seed bundle: %v", ErrInvalidBundle, err)
|
||
}
|
||
defer gz.Close()
|
||
|
||
payload, err := io.ReadAll(gz)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("%w: reading catalog seed bundle: %v", ErrInvalidBundle, err)
|
||
}
|
||
|
||
var bundle Bundle
|
||
if err := json.Unmarshal(payload, &bundle); err != nil {
|
||
return nil, fmt.Errorf("%w: decoding catalog seed bundle: %v", ErrInvalidBundle, err)
|
||
}
|
||
return &bundle, nil
|
||
}
|
||
|
||
func prepareLibraries(records []LibraryRecord, rewrites []PathRewrite) ([]LibraryRecord, map[int][]string, []string, error) {
|
||
libraries := make([]LibraryRecord, 0, len(records))
|
||
folderPaths := make(map[int][]string, len(records))
|
||
var unmatched []string
|
||
|
||
for _, record := range records {
|
||
finalPaths := make([]string, 0, len(record.Paths))
|
||
for _, path := range record.Paths {
|
||
rewritten, _ := rewritePath(path, rewrites)
|
||
rewritten = normalizePath(rewritten)
|
||
if !filepath.IsAbs(rewritten) {
|
||
return nil, nil, nil, fmt.Errorf("catalog seed path %q is not absolute after rewrite", rewritten)
|
||
}
|
||
finalPaths = append(finalPaths, rewritten)
|
||
if _, err := os.Stat(rewritten); err != nil {
|
||
if os.IsNotExist(err) {
|
||
unmatched = append(unmatched, rewritten)
|
||
continue
|
||
}
|
||
return nil, nil, nil, fmt.Errorf("checking library path %q: %w", rewritten, err)
|
||
}
|
||
}
|
||
sort.Strings(finalPaths)
|
||
record.Paths = dedupeStrings(finalPaths)
|
||
folderPaths[record.ExportedID] = append([]string(nil), record.Paths...)
|
||
libraries = append(libraries, record)
|
||
}
|
||
|
||
return libraries, folderPaths, dedupeStrings(unmatched), nil
|
||
}
|
||
|
||
func validateFileRoots(files []FileRecord, folderPaths map[int][]string, rewrites []PathRewrite) error {
|
||
var unmatched []string
|
||
for _, file := range files {
|
||
rewritten, matched := rewritePath(file.FilePath, rewrites)
|
||
rewritten = normalizePath(rewritten)
|
||
paths := folderPaths[file.MediaFolderID]
|
||
ok := false
|
||
for _, root := range paths {
|
||
if hasPathPrefix(rewritten, root) {
|
||
ok = true
|
||
break
|
||
}
|
||
}
|
||
if !ok {
|
||
if matched {
|
||
unmatched = append(unmatched, rewritten)
|
||
} else {
|
||
unmatched = append(unmatched, file.FilePath)
|
||
}
|
||
}
|
||
}
|
||
if len(unmatched) > 0 {
|
||
sort.Strings(unmatched)
|
||
return &UnmatchedRootsError{Roots: dedupeStrings(unmatched)}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func rewriteFileRecord(file FileRecord, rewrites []PathRewrite, folderPaths map[int][]string) (FileRecord, error) {
|
||
rewrittenPath, _ := rewritePath(file.FilePath, rewrites)
|
||
file.FilePath = normalizePath(rewrittenPath)
|
||
if !filepath.IsAbs(file.FilePath) {
|
||
return FileRecord{}, fmt.Errorf("catalog seed file path %q is not absolute after rewrite", file.FilePath)
|
||
}
|
||
for i := range file.ExternalSubtitles {
|
||
if file.ExternalSubtitles[i].Path == "" {
|
||
continue
|
||
}
|
||
subtitlePath, _ := rewritePath(file.ExternalSubtitles[i].Path, rewrites)
|
||
file.ExternalSubtitles[i].Path = normalizePath(subtitlePath)
|
||
if !filepath.IsAbs(file.ExternalSubtitles[i].Path) {
|
||
return FileRecord{}, fmt.Errorf("catalog seed subtitle path %q is not absolute after rewrite", file.ExternalSubtitles[i].Path)
|
||
}
|
||
}
|
||
|
||
paths := folderPaths[file.MediaFolderID]
|
||
for _, root := range paths {
|
||
if hasPathPrefix(file.FilePath, root) {
|
||
return file, nil
|
||
}
|
||
}
|
||
|
||
return FileRecord{}, &UnmatchedRootsError{Roots: []string{file.FilePath}}
|
||
}
|
||
|
||
func importLibrary(ctx context.Context, tx pgx.Tx, library LibraryRecord, overwrite bool) (localID int, created bool, matched bool, err error) {
|
||
existingIDs, err := lookupFolderIDsByPaths(ctx, tx, library.Paths)
|
||
if err != nil {
|
||
return 0, false, false, err
|
||
}
|
||
switch len(existingIDs) {
|
||
case 0:
|
||
err = tx.QueryRow(ctx, `
|
||
INSERT INTO media_folders (
|
||
type, name, enabled, last_scanned_at,
|
||
scan_warning_code, scan_warning_message, scan_warning_at, allow_empty_cleanup_once
|
||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||
RETURNING id`,
|
||
library.Type,
|
||
library.Name,
|
||
library.Enabled,
|
||
library.LastScannedAt,
|
||
library.ScanWarningCode,
|
||
library.ScanWarningMessage,
|
||
library.ScanWarningAt,
|
||
library.AllowEmptyCleanupOnce,
|
||
).Scan(&localID)
|
||
if err != nil {
|
||
return 0, false, false, fmt.Errorf("creating imported library %q: %w", library.Name, err)
|
||
}
|
||
for _, path := range library.Paths {
|
||
if _, err := tx.Exec(ctx,
|
||
`INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2)`,
|
||
localID, path,
|
||
); err != nil {
|
||
return 0, false, false, fmt.Errorf("creating imported library path %q: %w", path, err)
|
||
}
|
||
}
|
||
return localID, true, false, nil
|
||
case 1:
|
||
localID = existingIDs[0]
|
||
if overwrite {
|
||
if _, err := tx.Exec(ctx, `
|
||
UPDATE media_folders
|
||
SET type = $1,
|
||
name = $2,
|
||
enabled = $3,
|
||
last_scanned_at = $4,
|
||
scan_warning_code = $5,
|
||
scan_warning_message = $6,
|
||
scan_warning_at = $7,
|
||
allow_empty_cleanup_once = $8
|
||
WHERE id = $9`,
|
||
library.Type,
|
||
library.Name,
|
||
library.Enabled,
|
||
library.LastScannedAt,
|
||
library.ScanWarningCode,
|
||
library.ScanWarningMessage,
|
||
library.ScanWarningAt,
|
||
library.AllowEmptyCleanupOnce,
|
||
localID,
|
||
); err != nil {
|
||
return 0, false, false, fmt.Errorf("updating imported library %d: %w", localID, err)
|
||
}
|
||
}
|
||
for _, path := range library.Paths {
|
||
if _, err := tx.Exec(ctx,
|
||
`INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2) ON CONFLICT (path) DO NOTHING`,
|
||
localID, path,
|
||
); err != nil {
|
||
return 0, false, false, fmt.Errorf("upserting library path %q: %w", path, err)
|
||
}
|
||
}
|
||
return localID, false, true, nil
|
||
default:
|
||
return 0, false, false, fmt.Errorf("catalog seed paths for library %q map to multiple local libraries", library.Name)
|
||
}
|
||
}
|
||
|
||
const catalogImportBatchSize = 1000
|
||
|
||
type importPersonState struct {
|
||
ID int64
|
||
Name string
|
||
LowerName string
|
||
TmdbID string
|
||
ImdbID string
|
||
TvdbID string
|
||
PlexGUID string
|
||
PhotoPath string
|
||
PhotoThumbhash string
|
||
IsNew bool
|
||
NeedsUpdate bool
|
||
}
|
||
|
||
func isEmptyCatalogImportTarget(ctx context.Context, tx pgx.Tx) (bool, error) {
|
||
var empty bool
|
||
err := tx.QueryRow(ctx, `
|
||
SELECT
|
||
NOT EXISTS (SELECT 1 FROM media_folders) AND
|
||
NOT EXISTS (SELECT 1 FROM media_items) AND
|
||
NOT EXISTS (SELECT 1 FROM item_people) AND
|
||
NOT EXISTS (SELECT 1 FROM seasons) AND
|
||
NOT EXISTS (SELECT 1 FROM episodes) AND
|
||
NOT EXISTS (SELECT 1 FROM media_files) AND
|
||
NOT EXISTS (SELECT 1 FROM media_item_libraries)`).
|
||
Scan(&empty)
|
||
if err != nil {
|
||
return false, fmt.Errorf("checking empty catalog import target: %w", err)
|
||
}
|
||
return empty, nil
|
||
}
|
||
|
||
func bulkInsertItems(ctx context.Context, tx pgx.Tx, items []ItemRecord, onBatch func(processed int)) error {
|
||
rows := make([][]any, 0, len(items))
|
||
for _, item := range items {
|
||
rows = append(rows, []any{
|
||
item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres,
|
||
item.ContentRating, item.Runtime, item.Overview, item.Tagline,
|
||
item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience,
|
||
item.ImdbID, item.TmdbID, item.TvdbID,
|
||
item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath,
|
||
item.MetadataS3Path, item.MetadataEtag, item.SeasonCount,
|
||
item.Studios, item.Networks, item.Countries, item.Keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate,
|
||
item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status,
|
||
item.CreatedAt, item.UpdatedAt,
|
||
})
|
||
}
|
||
|
||
return executeInsertBatches(ctx, tx, `
|
||
INSERT INTO media_items (
|
||
content_id, type, title, sort_title, original_title, year, genres,
|
||
content_rating, runtime, overview, tagline,
|
||
rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience,
|
||
imdb_id, tmdb_id, tvdb_id,
|
||
poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path,
|
||
metadata_s3_path, metadata_etag, season_count,
|
||
studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date,
|
||
matched_at, last_refreshed, refresh_failures, locked_fields, status,
|
||
created_at, updated_at
|
||
) VALUES `,
|
||
rows,
|
||
41,
|
||
nil,
|
||
"",
|
||
onBatch,
|
||
)
|
||
}
|
||
|
||
func (s *Service) replacePeople(ctx context.Context, tx pgx.Tx, people []PersonRecord, itemStates map[string]bool, mode ConflictMode, result *ImportResult) error {
|
||
shouldReplace := make(map[string]struct{})
|
||
if itemStates == nil {
|
||
// Empty-catalog fast path: replace everything.
|
||
for _, p := range people {
|
||
shouldReplace[p.ContentID] = struct{}{}
|
||
}
|
||
} else {
|
||
for contentID, changed := range itemStates {
|
||
if mode == ConflictModeOverwrite || changed {
|
||
shouldReplace[contentID] = struct{}{}
|
||
}
|
||
}
|
||
}
|
||
|
||
if len(shouldReplace) == 0 {
|
||
return nil
|
||
}
|
||
|
||
contentIDs := mapKeys(shouldReplace)
|
||
sort.Strings(contentIDs)
|
||
if err := clearImportedPeople(ctx, tx, contentIDs); err != nil {
|
||
return err
|
||
}
|
||
|
||
filtered := make([]PersonRecord, 0, len(people))
|
||
tmdbIDs := make([]string, 0, len(people))
|
||
imdbIDs := make([]string, 0, len(people))
|
||
nameKeys := make([]string, 0, len(people))
|
||
for _, rec := range people {
|
||
if _, ok := shouldReplace[rec.ContentID]; !ok {
|
||
continue
|
||
}
|
||
filtered = append(filtered, rec)
|
||
if rec.TmdbID != "" {
|
||
tmdbIDs = append(tmdbIDs, rec.TmdbID)
|
||
}
|
||
if rec.ImdbID != "" {
|
||
imdbIDs = append(imdbIDs, rec.ImdbID)
|
||
}
|
||
if rec.Name != "" {
|
||
nameKeys = append(nameKeys, strings.ToLower(rec.Name))
|
||
}
|
||
}
|
||
|
||
if len(filtered) == 0 {
|
||
result.CreditsReplaced += len(contentIDs)
|
||
return nil
|
||
}
|
||
|
||
byTMDB, byIMDb, byName, err := loadExistingImportPeople(ctx, tx, tmdbIDs, imdbIDs, nameKeys)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
newPeople := make([]*importPersonState, 0)
|
||
updatedPeople := make(map[int64]*importPersonState)
|
||
|
||
type itemPersonKey struct {
|
||
contentID string
|
||
personID int64
|
||
kind int
|
||
character string
|
||
}
|
||
seenItemPeople := make(map[itemPersonKey]struct{}, len(filtered))
|
||
itemPeopleRows := make([][]any, 0, len(filtered))
|
||
for _, rec := range filtered {
|
||
person, resolveErr := resolveImportPerson(rec, byTMDB, byIMDb, byName, &newPeople, updatedPeople)
|
||
if resolveErr != nil {
|
||
return fmt.Errorf("resolving person %q for %s: %w", rec.Name, rec.ContentID, resolveErr)
|
||
}
|
||
|
||
// PostgreSQL's ON CONFLICT cannot handle the same conflict-target tuple
|
||
// appearing twice in one INSERT. Deduplicate after person resolution so
|
||
// that collisions caused by person-merging are also caught.
|
||
key := itemPersonKey{rec.ContentID, person.ID, rec.Kind, rec.Character}
|
||
if _, exists := seenItemPeople[key]; exists {
|
||
continue
|
||
}
|
||
seenItemPeople[key] = struct{}{}
|
||
|
||
itemPersonID, idErr := nextInt64ID()
|
||
if idErr != nil {
|
||
return fmt.Errorf("generating item_people id for %s: %w", rec.ContentID, idErr)
|
||
}
|
||
itemPeopleRows = append(itemPeopleRows, []any{
|
||
itemPersonID,
|
||
rec.ContentID,
|
||
person.ID,
|
||
rec.Kind,
|
||
rec.Character,
|
||
rec.SortOrder,
|
||
})
|
||
}
|
||
|
||
if err := bulkInsertPeople(ctx, tx, newPeople); err != nil {
|
||
return err
|
||
}
|
||
if err := bulkUpdatePeople(ctx, tx, updatedPeople); err != nil {
|
||
return err
|
||
}
|
||
if err := bulkInsertItemPeople(ctx, tx, itemPeopleRows); err != nil {
|
||
return err
|
||
}
|
||
|
||
result.CreditsReplaced += len(contentIDs)
|
||
return nil
|
||
}
|
||
|
||
// embeddingImportBatchSize is larger than the general batch size because
|
||
// embeddings only have 4 columns (4 × 5000 = 20,000 params, well within the 65,535 limit).
|
||
const embeddingImportBatchSize = 5000
|
||
|
||
func (s *Service) importEmbeddings(ctx context.Context, tx pgx.Tx, embeddings []EmbeddingRecord, emptyTarget bool, result *ImportResult, onBatch func(int)) error {
|
||
if len(embeddings) == 0 {
|
||
return nil
|
||
}
|
||
|
||
// For empty targets, drop the HNSW index before bulk loading and recreate after.
|
||
// Maintaining an HNSW graph incrementally during inserts is far slower than building
|
||
// it once on the complete dataset.
|
||
if emptyTarget {
|
||
if _, err := tx.Exec(ctx, `DROP INDEX IF EXISTS idx_media_item_embeddings_hnsw`); err != nil {
|
||
return fmt.Errorf("dropping embedding index for bulk load: %w", err)
|
||
}
|
||
}
|
||
|
||
rows := make([][]any, 0, len(embeddings))
|
||
for _, rec := range embeddings {
|
||
padded := rec.Embedding
|
||
if len(padded) > recommendations.CanonicalEmbeddingDimensions {
|
||
return fmt.Errorf(
|
||
"embedding vector length %d exceeds canonical dimension %d for media item %s",
|
||
len(padded),
|
||
recommendations.CanonicalEmbeddingDimensions,
|
||
rec.MediaItemID,
|
||
)
|
||
}
|
||
if len(padded) < recommendations.CanonicalEmbeddingDimensions {
|
||
p := make([]float32, recommendations.CanonicalEmbeddingDimensions)
|
||
copy(p, padded)
|
||
padded = p
|
||
}
|
||
rows = append(rows, []any{
|
||
rec.MediaItemID,
|
||
pgvector.NewVector(padded),
|
||
rec.Model,
|
||
rec.CanonicalText,
|
||
})
|
||
}
|
||
|
||
batchCallback := func(processed int) {
|
||
result.EmbeddingsImported += processed
|
||
if onBatch != nil {
|
||
onBatch(processed)
|
||
}
|
||
}
|
||
|
||
var suffix string
|
||
if emptyTarget {
|
||
suffix = ""
|
||
} else {
|
||
suffix = `
|
||
ON CONFLICT (media_item_id) DO UPDATE
|
||
SET embedding = EXCLUDED.embedding,
|
||
model = EXCLUDED.model,
|
||
canonical_text = EXCLUDED.canonical_text,
|
||
updated_at = NOW()`
|
||
}
|
||
|
||
for start := 0; start < len(rows); start += embeddingImportBatchSize {
|
||
end := min(start+embeddingImportBatchSize, len(rows))
|
||
|
||
var builder strings.Builder
|
||
builder.WriteString(`
|
||
INSERT INTO media_item_embeddings (media_item_id, embedding, model, canonical_text)
|
||
VALUES `)
|
||
args := make([]any, 0, (end-start)*4)
|
||
for rowIndex := start; rowIndex < end; rowIndex++ {
|
||
if rowIndex > start {
|
||
builder.WriteString(", ")
|
||
}
|
||
builder.WriteString(buildValuesClause(len(args)+1, 4, nil))
|
||
args = append(args, rows[rowIndex]...)
|
||
}
|
||
builder.WriteString(suffix)
|
||
|
||
if _, err := tx.Exec(ctx, builder.String(), args...); err != nil {
|
||
return fmt.Errorf("inserting embeddings batch: %w", err)
|
||
}
|
||
batchCallback(end - start)
|
||
}
|
||
|
||
// Recreate the HNSW index after bulk load.
|
||
if emptyTarget {
|
||
if _, err := tx.Exec(ctx, fmt.Sprintf(
|
||
`CREATE INDEX idx_media_item_embeddings_hnsw ON media_item_embeddings USING hnsw ((embedding::halfvec(%d)) halfvec_cosine_ops)`,
|
||
recommendations.CanonicalEmbeddingDimensions,
|
||
)); err != nil {
|
||
return fmt.Errorf("recreating embedding index after bulk load: %w", err)
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
func bulkInsertSeasons(ctx context.Context, tx pgx.Tx, seasons []SeasonRecord, onBatch func(processed int)) error {
|
||
rows := make([][]any, 0, len(seasons))
|
||
for _, season := range seasons {
|
||
rows = append(rows, []any{
|
||
season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate,
|
||
season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag,
|
||
season.CreatedAt, season.UpdatedAt,
|
||
})
|
||
}
|
||
|
||
return executeInsertBatches(ctx, tx, `
|
||
INSERT INTO seasons (
|
||
content_id, series_id, season_number, title, overview, air_date,
|
||
poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES `,
|
||
rows,
|
||
12,
|
||
nil,
|
||
"",
|
||
onBatch,
|
||
)
|
||
}
|
||
|
||
func bulkInsertEpisodes(ctx context.Context, tx pgx.Tx, episodes []EpisodeRecord, onBatch func(processed int)) error {
|
||
rows := make([][]any, 0, len(episodes))
|
||
for _, episode := range episodes {
|
||
rows = append(rows, []any{
|
||
episode.ContentID, episode.SeriesID, nullableString(episode.SeasonID), episode.SeasonNumber, episode.EpisodeNumber,
|
||
episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB,
|
||
episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag,
|
||
episode.CreatedAt, episode.UpdatedAt,
|
||
})
|
||
}
|
||
|
||
return executeInsertBatches(ctx, tx, `
|
||
INSERT INTO episodes (
|
||
content_id, series_id, season_id, season_number, episode_number,
|
||
title, overview, air_date, runtime, rating_imdb, rating_tmdb,
|
||
still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES `,
|
||
rows,
|
||
17,
|
||
nil,
|
||
"",
|
||
onBatch,
|
||
)
|
||
}
|
||
|
||
func bulkInsertFiles(ctx context.Context, tx pgx.Tx, files []FileRecord, onBatch func(processed int)) error {
|
||
rows := make([][]any, 0, len(files))
|
||
for _, file := range files {
|
||
videoTracksJSON, err := json.Marshal(file.VideoTracks)
|
||
if err != nil {
|
||
return fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, err)
|
||
}
|
||
audioTracksJSON, err := json.Marshal(file.AudioTracks)
|
||
if err != nil {
|
||
return fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, err)
|
||
}
|
||
subtitleTracksJSON, err := json.Marshal(file.SubtitleTracks)
|
||
if err != nil {
|
||
return fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, err)
|
||
}
|
||
externalSubtitlesJSON, err := json.Marshal(file.ExternalSubtitles)
|
||
if err != nil {
|
||
return fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, err)
|
||
}
|
||
rows = append(rows, []any{
|
||
nullableString(file.ContentID), nullableString(file.EpisodeID), nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber),
|
||
file.MediaFolderID, file.FilePath, file.FileSize, nullableString(file.FileHash),
|
||
file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container,
|
||
nilIfZero(file.Duration), nilIfZero(file.Bitrate), string(videoTracksJSON), string(audioTracksJSON), string(subtitleTracksJSON), string(externalSubtitlesJSON),
|
||
file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd,
|
||
nullableString(file.ProbeSource), file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt,
|
||
})
|
||
}
|
||
|
||
return executeInsertBatches(ctx, tx, `
|
||
INSERT INTO media_files (
|
||
content_id, episode_id, season_number, episode_number,
|
||
media_folder_id, file_path, file_size, file_hash,
|
||
codec_video, codec_audio, resolution, audio_channels, hdr, container,
|
||
duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles,
|
||
intro_start, intro_end, credits_start, credits_end,
|
||
probe_source, probe_updated_at, missing_since, created_at, updated_at
|
||
) VALUES `,
|
||
rows,
|
||
29,
|
||
map[int]string{
|
||
16: "::jsonb",
|
||
17: "::jsonb",
|
||
18: "::jsonb",
|
||
19: "::jsonb",
|
||
},
|
||
"",
|
||
onBatch,
|
||
)
|
||
}
|
||
|
||
func bulkInsertLibraryLinks(ctx context.Context, tx pgx.Tx, links []LibraryLinkRecord, onBatch func(processed int)) error {
|
||
rows := make([][]any, 0, len(links))
|
||
for _, link := range links {
|
||
rows = append(rows, []any{link.ContentID, link.MediaFolderID, link.FirstSeenAt})
|
||
}
|
||
|
||
return executeInsertBatches(ctx, tx, `
|
||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||
VALUES `,
|
||
rows,
|
||
3,
|
||
nil,
|
||
"",
|
||
onBatch,
|
||
)
|
||
}
|
||
|
||
func executeInsertBatches(ctx context.Context, tx pgx.Tx, prefix string, rows [][]any, columnsPerRow int, casts map[int]string, suffix string, onBatch func(processed int)) error {
|
||
_, err := executeInsertBatchesCounted(ctx, tx, prefix, rows, columnsPerRow, casts, suffix, onBatch)
|
||
return err
|
||
}
|
||
|
||
// executeInsertBatchesCounted runs batched inserts and returns total rows affected.
|
||
// For ON CONFLICT DO NOTHING, RowsAffected = rows actually inserted.
|
||
// For ON CONFLICT DO UPDATE, RowsAffected = all rows (inserts + updates).
|
||
func executeInsertBatchesCounted(ctx context.Context, tx pgx.Tx, prefix string, rows [][]any, columnsPerRow int, casts map[int]string, suffix string, onBatch func(processed int)) (int64, error) {
|
||
if len(rows) == 0 {
|
||
return 0, nil
|
||
}
|
||
|
||
var total int64
|
||
for start := 0; start < len(rows); start += catalogImportBatchSize {
|
||
end := min(start+catalogImportBatchSize, len(rows))
|
||
|
||
var builder strings.Builder
|
||
builder.WriteString(prefix)
|
||
args := make([]any, 0, (end-start)*columnsPerRow)
|
||
for rowIndex := start; rowIndex < end; rowIndex++ {
|
||
if rowIndex > start {
|
||
builder.WriteString(", ")
|
||
}
|
||
builder.WriteString(buildValuesClause(len(args)+1, columnsPerRow, casts))
|
||
args = append(args, rows[rowIndex]...)
|
||
}
|
||
builder.WriteString(suffix)
|
||
|
||
tag, err := tx.Exec(ctx, builder.String(), args...)
|
||
if err != nil {
|
||
return total, err
|
||
}
|
||
total += tag.RowsAffected()
|
||
if onBatch != nil {
|
||
onBatch(end - start)
|
||
}
|
||
}
|
||
|
||
return total, nil
|
||
}
|
||
|
||
type batchUpsertCounts struct {
|
||
Created int
|
||
Updated int
|
||
Skipped int
|
||
}
|
||
|
||
// executeUpsertBatches runs batched inserts with ON CONFLICT handling and returns
|
||
// accurate created/updated/skipped counts. For skip_existing mode it uses Exec
|
||
// with RowsAffected; for overwrite mode it uses RETURNING (xmax = 0) to
|
||
// distinguish inserts from updates.
|
||
func executeUpsertBatches(ctx context.Context, tx pgx.Tx, prefix string, rows [][]any, columnsPerRow int, casts map[int]string, skipSuffix, upsertSuffix string, mode ConflictMode, onBatch func(processed int)) (batchUpsertCounts, error) {
|
||
var counts batchUpsertCounts
|
||
if len(rows) == 0 {
|
||
return counts, nil
|
||
}
|
||
|
||
if mode == ConflictModeSkipExisting {
|
||
affected, err := executeInsertBatchesCounted(ctx, tx, prefix, rows, columnsPerRow, casts, skipSuffix, onBatch)
|
||
if err != nil {
|
||
return counts, err
|
||
}
|
||
counts.Created = int(affected)
|
||
counts.Skipped = len(rows) - counts.Created
|
||
return counts, nil
|
||
}
|
||
|
||
// Overwrite mode: append RETURNING (xmax = 0) to distinguish inserts from updates.
|
||
fullSuffix := upsertSuffix + ` RETURNING (xmax = 0)`
|
||
for start := 0; start < len(rows); start += catalogImportBatchSize {
|
||
end := min(start+catalogImportBatchSize, len(rows))
|
||
|
||
var builder strings.Builder
|
||
builder.WriteString(prefix)
|
||
args := make([]any, 0, (end-start)*columnsPerRow)
|
||
for rowIndex := start; rowIndex < end; rowIndex++ {
|
||
if rowIndex > start {
|
||
builder.WriteString(", ")
|
||
}
|
||
builder.WriteString(buildValuesClause(len(args)+1, columnsPerRow, casts))
|
||
args = append(args, rows[rowIndex]...)
|
||
}
|
||
builder.WriteString(fullSuffix)
|
||
|
||
queryRows, queryErr := tx.Query(ctx, builder.String(), args...)
|
||
if queryErr != nil {
|
||
return counts, queryErr
|
||
}
|
||
for queryRows.Next() {
|
||
var isNew bool
|
||
if scanErr := queryRows.Scan(&isNew); scanErr != nil {
|
||
queryRows.Close()
|
||
return counts, scanErr
|
||
}
|
||
if isNew {
|
||
counts.Created++
|
||
} else {
|
||
counts.Updated++
|
||
}
|
||
}
|
||
queryRows.Close()
|
||
if err := queryRows.Err(); err != nil {
|
||
return counts, err
|
||
}
|
||
|
||
if onBatch != nil {
|
||
onBatch(end - start)
|
||
}
|
||
}
|
||
|
||
return counts, nil
|
||
}
|
||
|
||
const copyBatchSize = 10000
|
||
|
||
// copyInsertBatches uses PostgreSQL's COPY protocol for fast bulk loading.
|
||
// Much faster than multi-row INSERT as it bypasses query parsing/planning.
|
||
// Only suitable for empty tables (no ON CONFLICT support).
|
||
func copyInsertBatches(ctx context.Context, tx pgx.Tx, table string, columns []string, rows [][]any, onBatch func(int)) error {
|
||
if len(rows) == 0 {
|
||
return nil
|
||
}
|
||
|
||
for start := 0; start < len(rows); start += copyBatchSize {
|
||
end := min(start+copyBatchSize, len(rows))
|
||
_, err := tx.CopyFrom(ctx,
|
||
pgx.Identifier{table},
|
||
columns,
|
||
pgx.CopyFromRows(rows[start:end]),
|
||
)
|
||
if err != nil {
|
||
return fmt.Errorf("copy into %s: %w", table, err)
|
||
}
|
||
if onBatch != nil {
|
||
onBatch(end - start)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func buildValuesClause(base, columns int, casts map[int]string) string {
|
||
parts := make([]string, 0, columns)
|
||
for i := range columns {
|
||
part := fmt.Sprintf("$%d", base+i)
|
||
if cast, ok := casts[i]; ok {
|
||
part += cast
|
||
}
|
||
parts = append(parts, part)
|
||
}
|
||
return "(" + strings.Join(parts, ", ") + ")"
|
||
}
|
||
|
||
func loadExistingImportPeople(ctx context.Context, tx pgx.Tx, tmdbIDs, imdbIDs, nameKeys []string) (map[string]*importPersonState, map[string]*importPersonState, map[string]*importPersonState, error) {
|
||
byTMDB := make(map[string]*importPersonState)
|
||
byIMDb := make(map[string]*importPersonState)
|
||
byName := make(map[string]*importPersonState)
|
||
statesByID := make(map[int64]*importPersonState)
|
||
|
||
register := func(state importPersonState) {
|
||
canonical := statesByID[state.ID]
|
||
if canonical == nil {
|
||
copied := state
|
||
canonical = &copied
|
||
statesByID[state.ID] = canonical
|
||
}
|
||
if canonical.Name == "" {
|
||
canonical.Name = state.Name
|
||
canonical.LowerName = state.LowerName
|
||
}
|
||
if canonical.TmdbID == "" {
|
||
canonical.TmdbID = state.TmdbID
|
||
}
|
||
if canonical.ImdbID == "" {
|
||
canonical.ImdbID = state.ImdbID
|
||
}
|
||
if canonical.TvdbID == "" {
|
||
canonical.TvdbID = state.TvdbID
|
||
}
|
||
if canonical.PlexGUID == "" {
|
||
canonical.PlexGUID = state.PlexGUID
|
||
}
|
||
if canonical.PhotoPath == "" {
|
||
canonical.PhotoPath = state.PhotoPath
|
||
}
|
||
if canonical.PhotoThumbhash == "" {
|
||
canonical.PhotoThumbhash = state.PhotoThumbhash
|
||
}
|
||
registerImportPerson(canonical, byTMDB, byIMDb, byName)
|
||
}
|
||
|
||
if err := queryImportPeople(ctx, tx, `
|
||
SELECT id, name, LOWER(name), tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash
|
||
FROM people
|
||
WHERE tmdb_id = ANY($1)`,
|
||
dedupeStrings(tmdbIDs), register,
|
||
); err != nil {
|
||
return nil, nil, nil, err
|
||
}
|
||
if err := queryImportPeople(ctx, tx, `
|
||
SELECT id, name, LOWER(name), tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash
|
||
FROM people
|
||
WHERE imdb_id = ANY($1)`,
|
||
dedupeStrings(imdbIDs), register,
|
||
); err != nil {
|
||
return nil, nil, nil, err
|
||
}
|
||
if err := queryImportPeople(ctx, tx, `
|
||
SELECT DISTINCT ON (LOWER(name))
|
||
id, name, LOWER(name), tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash
|
||
FROM people
|
||
WHERE LOWER(name) = ANY($1)
|
||
ORDER BY LOWER(name), id`,
|
||
dedupeStrings(nameKeys), register,
|
||
); err != nil {
|
||
return nil, nil, nil, err
|
||
}
|
||
|
||
return byTMDB, byIMDb, byName, nil
|
||
}
|
||
|
||
func queryImportPeople(ctx context.Context, tx pgx.Tx, query string, keys []string, add func(importPersonState)) error {
|
||
if len(keys) == 0 {
|
||
return nil
|
||
}
|
||
|
||
rows, err := tx.Query(ctx, query, keys)
|
||
if err != nil {
|
||
return fmt.Errorf("loading imported people matches: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
for rows.Next() {
|
||
var state importPersonState
|
||
if err := rows.Scan(
|
||
&state.ID,
|
||
&state.Name,
|
||
&state.LowerName,
|
||
&state.TmdbID,
|
||
&state.ImdbID,
|
||
&state.TvdbID,
|
||
&state.PlexGUID,
|
||
&state.PhotoPath,
|
||
&state.PhotoThumbhash,
|
||
); err != nil {
|
||
return fmt.Errorf("scanning imported person match: %w", err)
|
||
}
|
||
add(state)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return fmt.Errorf("iterating imported person matches: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func resolveImportPerson(rec PersonRecord, byTMDB, byIMDb, byName map[string]*importPersonState, newPeople *[]*importPersonState, updatedPeople map[int64]*importPersonState) (*importPersonState, error) {
|
||
nameKey := strings.ToLower(rec.Name)
|
||
person := findImportPerson(rec, nameKey, byTMDB, byIMDb, byName)
|
||
if person == nil {
|
||
personID, err := nextInt64ID()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
person = &importPersonState{
|
||
ID: personID,
|
||
Name: rec.Name,
|
||
LowerName: nameKey,
|
||
TmdbID: rec.TmdbID,
|
||
ImdbID: rec.ImdbID,
|
||
TvdbID: rec.TvdbID,
|
||
PlexGUID: rec.PlexGUID,
|
||
PhotoPath: rec.PhotoPath,
|
||
PhotoThumbhash: rec.PhotoThumbhash,
|
||
IsNew: true,
|
||
}
|
||
*newPeople = append(*newPeople, person)
|
||
registerImportPerson(person, byTMDB, byIMDb, byName)
|
||
return person, nil
|
||
}
|
||
|
||
enrichImportPerson(person, rec)
|
||
registerImportPerson(person, byTMDB, byIMDb, byName)
|
||
if person.NeedsUpdate {
|
||
updatedPeople[person.ID] = person
|
||
}
|
||
return person, nil
|
||
}
|
||
|
||
func findImportPerson(rec PersonRecord, nameKey string, byTMDB, byIMDb, byName map[string]*importPersonState) *importPersonState {
|
||
if rec.TmdbID != "" {
|
||
if person := byTMDB[rec.TmdbID]; person != nil {
|
||
return person
|
||
}
|
||
}
|
||
if rec.ImdbID != "" {
|
||
if person := byIMDb[rec.ImdbID]; person != nil {
|
||
return person
|
||
}
|
||
}
|
||
if nameKey != "" {
|
||
if person := byName[nameKey]; person != nil {
|
||
return person
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func enrichImportPerson(person *importPersonState, rec PersonRecord) {
|
||
if person.TmdbID == "" && rec.TmdbID != "" {
|
||
person.TmdbID = rec.TmdbID
|
||
person.NeedsUpdate = person.NeedsUpdate || !person.IsNew
|
||
}
|
||
if person.ImdbID == "" && rec.ImdbID != "" {
|
||
person.ImdbID = rec.ImdbID
|
||
person.NeedsUpdate = person.NeedsUpdate || !person.IsNew
|
||
}
|
||
if person.TvdbID == "" && rec.TvdbID != "" {
|
||
person.TvdbID = rec.TvdbID
|
||
person.NeedsUpdate = person.NeedsUpdate || !person.IsNew
|
||
}
|
||
if person.PlexGUID == "" && rec.PlexGUID != "" {
|
||
person.PlexGUID = rec.PlexGUID
|
||
person.NeedsUpdate = person.NeedsUpdate || !person.IsNew
|
||
}
|
||
if person.PhotoPath == "" && rec.PhotoPath != "" {
|
||
person.PhotoPath = rec.PhotoPath
|
||
person.NeedsUpdate = person.NeedsUpdate || !person.IsNew
|
||
}
|
||
if person.PhotoThumbhash == "" && rec.PhotoThumbhash != "" {
|
||
person.PhotoThumbhash = rec.PhotoThumbhash
|
||
person.NeedsUpdate = person.NeedsUpdate || !person.IsNew
|
||
}
|
||
}
|
||
|
||
func registerImportPerson(person *importPersonState, byTMDB, byIMDb, byName map[string]*importPersonState) {
|
||
if person.TmdbID != "" {
|
||
byTMDB[person.TmdbID] = person
|
||
}
|
||
if person.ImdbID != "" {
|
||
byIMDb[person.ImdbID] = person
|
||
}
|
||
if person.LowerName != "" {
|
||
byName[person.LowerName] = person
|
||
}
|
||
}
|
||
|
||
func clearImportedPeople(ctx context.Context, tx pgx.Tx, contentIDs []string) error {
|
||
if len(contentIDs) == 0 {
|
||
return nil
|
||
}
|
||
if _, err := tx.Exec(ctx, `DELETE FROM item_people WHERE content_id = ANY($1)`, contentIDs); err != nil {
|
||
return fmt.Errorf("clearing imported people: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func bulkInsertPeople(ctx context.Context, tx pgx.Tx, people []*importPersonState) error {
|
||
rows := make([][]any, 0, len(people))
|
||
for _, person := range people {
|
||
rows = append(rows, []any{
|
||
person.ID,
|
||
person.Name,
|
||
"",
|
||
"",
|
||
nil,
|
||
nil,
|
||
"",
|
||
"",
|
||
person.PhotoPath,
|
||
person.PhotoThumbhash,
|
||
person.TmdbID,
|
||
person.ImdbID,
|
||
person.TvdbID,
|
||
person.PlexGUID,
|
||
})
|
||
}
|
||
|
||
if err := executeInsertBatches(ctx, tx, `
|
||
INSERT INTO people (
|
||
id, name, sort_name, bio, birth_date, death_date, birthplace, homepage,
|
||
photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid
|
||
) VALUES `,
|
||
rows,
|
||
14,
|
||
nil,
|
||
"",
|
||
nil,
|
||
); err != nil {
|
||
return fmt.Errorf("inserting imported people: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func bulkUpdatePeople(ctx context.Context, tx pgx.Tx, people map[int64]*importPersonState) error {
|
||
if len(people) == 0 {
|
||
return nil
|
||
}
|
||
|
||
ids := make([]int64, 0, len(people))
|
||
for id := range people {
|
||
ids = append(ids, id)
|
||
}
|
||
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
|
||
|
||
rows := make([][]any, 0, len(ids))
|
||
for _, id := range ids {
|
||
person := people[id]
|
||
rows = append(rows, []any{
|
||
person.ID,
|
||
person.TmdbID,
|
||
person.ImdbID,
|
||
person.TvdbID,
|
||
person.PlexGUID,
|
||
person.PhotoPath,
|
||
person.PhotoThumbhash,
|
||
})
|
||
}
|
||
|
||
if err := executeInsertBatches(ctx, tx, `
|
||
UPDATE people AS p
|
||
SET tmdb_id = v.tmdb_id,
|
||
imdb_id = v.imdb_id,
|
||
tvdb_id = v.tvdb_id,
|
||
plex_guid = v.plex_guid,
|
||
photo_path = v.photo_path,
|
||
photo_thumbhash = v.photo_thumbhash,
|
||
updated_at = NOW()
|
||
FROM (VALUES `,
|
||
rows,
|
||
7,
|
||
nil,
|
||
`) AS v(id, tmdb_id, imdb_id, tvdb_id, plex_guid, photo_path, photo_thumbhash)
|
||
WHERE p.id = v.id`,
|
||
nil,
|
||
); err != nil {
|
||
return fmt.Errorf("updating imported people: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func bulkInsertItemPeople(ctx context.Context, tx pgx.Tx, rows [][]any) error {
|
||
if err := executeInsertBatches(ctx, tx, `
|
||
INSERT INTO item_people (id, content_id, person_id, kind, character, sort_order)
|
||
VALUES `,
|
||
rows,
|
||
6,
|
||
nil,
|
||
`
|
||
ON CONFLICT (content_id, person_id, kind, character) DO UPDATE
|
||
SET sort_order = EXCLUDED.sort_order`,
|
||
nil,
|
||
); err != nil {
|
||
return fmt.Errorf("inserting imported item_people: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func nextInt64ID() (int64, error) {
|
||
idStr, err := idgen.NextID()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
id, err := strconv.ParseInt(idStr, 10, 64)
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
return id, nil
|
||
}
|
||
|
||
func lookupFolderIDsByPaths(ctx context.Context, tx pgx.Tx, paths []string) ([]int, error) {
|
||
if len(paths) == 0 {
|
||
return nil, nil
|
||
}
|
||
rows, err := tx.Query(ctx, `SELECT DISTINCT media_folder_id FROM media_folder_paths WHERE path = ANY($1)`, paths)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("loading existing folder paths: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
var ids []int
|
||
for rows.Next() {
|
||
var id int
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, fmt.Errorf("scanning existing folder id: %w", err)
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
sort.Ints(ids)
|
||
return ids, rows.Err()
|
||
}
|
||
|
||
// batchImportItems inserts items in multi-row batches with ON CONFLICT handling.
|
||
// Returns a map of content_id → changed (true if created or updated) for people import,
|
||
// plus aggregate created/updated/skipped counts.
|
||
func batchImportItems(ctx context.Context, tx pgx.Tx, items []ItemRecord, mode ConflictMode, onBatch func(processed int)) (itemStates map[string]bool, created, updated, skipped int, err error) {
|
||
itemStates = make(map[string]bool, len(items))
|
||
if len(items) == 0 {
|
||
return itemStates, 0, 0, 0, nil
|
||
}
|
||
|
||
rows := make([][]any, 0, len(items))
|
||
contentIDs := make([]string, 0, len(items))
|
||
for _, item := range items {
|
||
studios, networks, countries, keywords := itemRecordStringArrays(item)
|
||
contentIDs = append(contentIDs, item.ContentID)
|
||
rows = append(rows, []any{
|
||
item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres,
|
||
item.ContentRating, item.Runtime, item.Overview, item.Tagline,
|
||
item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience,
|
||
item.ImdbID, item.TmdbID, item.TvdbID,
|
||
item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath,
|
||
item.MetadataS3Path, item.MetadataEtag, item.SeasonCount,
|
||
studios, networks, countries, keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate,
|
||
item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status,
|
||
item.CreatedAt, item.UpdatedAt,
|
||
})
|
||
}
|
||
|
||
const colCount = 41
|
||
prefix := `
|
||
INSERT INTO media_items (
|
||
content_id, type, title, sort_title, original_title, year, genres,
|
||
content_rating, runtime, overview, tagline,
|
||
rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience,
|
||
imdb_id, tmdb_id, tvdb_id,
|
||
poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path,
|
||
metadata_s3_path, metadata_etag, season_count,
|
||
studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date,
|
||
matched_at, last_refreshed, refresh_failures, locked_fields, status,
|
||
created_at, updated_at
|
||
) VALUES `
|
||
|
||
var suffix string
|
||
if mode == ConflictModeSkipExisting {
|
||
suffix = ` ON CONFLICT (content_id) DO NOTHING RETURNING content_id`
|
||
} else {
|
||
suffix = `
|
||
ON CONFLICT (content_id) DO UPDATE SET
|
||
type = EXCLUDED.type,
|
||
title = EXCLUDED.title,
|
||
sort_title = EXCLUDED.sort_title,
|
||
original_title = EXCLUDED.original_title,
|
||
year = EXCLUDED.year,
|
||
genres = EXCLUDED.genres,
|
||
content_rating = EXCLUDED.content_rating,
|
||
runtime = EXCLUDED.runtime,
|
||
overview = EXCLUDED.overview,
|
||
tagline = EXCLUDED.tagline,
|
||
rating_imdb = EXCLUDED.rating_imdb,
|
||
rating_tmdb = EXCLUDED.rating_tmdb,
|
||
rating_rt_critic = EXCLUDED.rating_rt_critic,
|
||
rating_rt_audience = EXCLUDED.rating_rt_audience,
|
||
imdb_id = EXCLUDED.imdb_id,
|
||
tmdb_id = EXCLUDED.tmdb_id,
|
||
tvdb_id = EXCLUDED.tvdb_id,
|
||
poster_path = EXCLUDED.poster_path,
|
||
poster_thumbhash = EXCLUDED.poster_thumbhash,
|
||
backdrop_path = EXCLUDED.backdrop_path,
|
||
backdrop_thumbhash = EXCLUDED.backdrop_thumbhash,
|
||
logo_path = EXCLUDED.logo_path,
|
||
metadata_s3_path = EXCLUDED.metadata_s3_path,
|
||
metadata_etag = EXCLUDED.metadata_etag,
|
||
season_count = EXCLUDED.season_count,
|
||
studios = EXCLUDED.studios,
|
||
networks = EXCLUDED.networks,
|
||
countries = EXCLUDED.countries,
|
||
keywords = EXCLUDED.keywords,
|
||
original_language = EXCLUDED.original_language,
|
||
release_date = EXCLUDED.release_date,
|
||
first_air_date = EXCLUDED.first_air_date,
|
||
last_air_date = EXCLUDED.last_air_date,
|
||
matched_at = EXCLUDED.matched_at,
|
||
last_refreshed = EXCLUDED.last_refreshed,
|
||
refresh_failures = EXCLUDED.refresh_failures,
|
||
locked_fields = EXCLUDED.locked_fields,
|
||
status = EXCLUDED.status,
|
||
updated_at = EXCLUDED.updated_at
|
||
RETURNING content_id, (xmax = 0)`
|
||
}
|
||
|
||
for start := 0; start < len(rows); start += catalogImportBatchSize {
|
||
end := min(start+catalogImportBatchSize, len(rows))
|
||
|
||
var builder strings.Builder
|
||
builder.WriteString(prefix)
|
||
args := make([]any, 0, (end-start)*colCount)
|
||
for rowIndex := start; rowIndex < end; rowIndex++ {
|
||
if rowIndex > start {
|
||
builder.WriteString(", ")
|
||
}
|
||
builder.WriteString(buildValuesClause(len(args)+1, colCount, nil))
|
||
args = append(args, rows[rowIndex]...)
|
||
}
|
||
builder.WriteString(suffix)
|
||
|
||
queryRows, queryErr := tx.Query(ctx, builder.String(), args...)
|
||
if queryErr != nil {
|
||
return nil, 0, 0, 0, fmt.Errorf("batch importing items: %w", queryErr)
|
||
}
|
||
|
||
if mode == ConflictModeSkipExisting {
|
||
// RETURNING content_id — only returned for actually-inserted rows.
|
||
createdSet := make(map[string]struct{})
|
||
for queryRows.Next() {
|
||
var contentID string
|
||
if scanErr := queryRows.Scan(&contentID); scanErr != nil {
|
||
queryRows.Close()
|
||
return nil, 0, 0, 0, fmt.Errorf("scanning imported item id: %w", scanErr)
|
||
}
|
||
createdSet[contentID] = struct{}{}
|
||
}
|
||
queryRows.Close()
|
||
if queryErr = queryRows.Err(); queryErr != nil {
|
||
return nil, 0, 0, 0, fmt.Errorf("iterating imported items: %w", queryErr)
|
||
}
|
||
for _, cid := range contentIDs[start:end] {
|
||
if _, ok := createdSet[cid]; ok {
|
||
itemStates[cid] = true
|
||
created++
|
||
} else {
|
||
skipped++
|
||
}
|
||
}
|
||
} else {
|
||
// RETURNING content_id, (xmax = 0) — returns for every row.
|
||
for queryRows.Next() {
|
||
var contentID string
|
||
var isNew bool
|
||
if scanErr := queryRows.Scan(&contentID, &isNew); scanErr != nil {
|
||
queryRows.Close()
|
||
return nil, 0, 0, 0, fmt.Errorf("scanning upserted item: %w", scanErr)
|
||
}
|
||
itemStates[contentID] = true
|
||
if isNew {
|
||
created++
|
||
} else {
|
||
updated++
|
||
}
|
||
}
|
||
queryRows.Close()
|
||
if queryErr = queryRows.Err(); queryErr != nil {
|
||
return nil, 0, 0, 0, fmt.Errorf("iterating upserted items: %w", queryErr)
|
||
}
|
||
}
|
||
|
||
if onBatch != nil {
|
||
onBatch(end - start)
|
||
}
|
||
}
|
||
|
||
return itemStates, created, updated, skipped, nil
|
||
}
|
||
|
||
func importItem(ctx context.Context, tx pgx.Tx, item ItemRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) {
|
||
if mode == ConflictModeSkipExisting {
|
||
tag, execErr := tx.Exec(ctx, `
|
||
INSERT INTO media_items (
|
||
content_id, type, title, sort_title, original_title, year, genres,
|
||
content_rating, runtime, overview, tagline,
|
||
rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience,
|
||
imdb_id, tmdb_id, tvdb_id,
|
||
poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path,
|
||
metadata_s3_path, metadata_etag, season_count,
|
||
studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date,
|
||
matched_at, last_refreshed, refresh_failures, locked_fields, status,
|
||
created_at, updated_at
|
||
) VALUES (
|
||
$1, $2, $3, $4, $5, $6, $7,
|
||
$8, $9, $10, $11,
|
||
$12, $13, $14, $15,
|
||
$16, $17, $18,
|
||
$19, $20, $21, $22, $23,
|
||
$24, $25, $26,
|
||
$27, $28, $29, $30, $31, $32, $33, $34,
|
||
$35, $36, $37, $38, $39,
|
||
$40, $41
|
||
)
|
||
ON CONFLICT (content_id) DO NOTHING`,
|
||
item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres,
|
||
item.ContentRating, item.Runtime, item.Overview, item.Tagline,
|
||
item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience,
|
||
item.ImdbID, item.TmdbID, item.TvdbID,
|
||
item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath,
|
||
item.MetadataS3Path, item.MetadataEtag, item.SeasonCount,
|
||
item.Studios, item.Networks, item.Countries, item.Keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate,
|
||
item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status,
|
||
item.CreatedAt, item.UpdatedAt,
|
||
)
|
||
if execErr != nil {
|
||
return false, false, false, fmt.Errorf("importing item %s: %w", item.ContentID, execErr)
|
||
}
|
||
return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil
|
||
}
|
||
|
||
err = tx.QueryRow(ctx, `
|
||
INSERT INTO media_items (
|
||
content_id, type, title, sort_title, original_title, year, genres,
|
||
content_rating, runtime, overview, tagline,
|
||
rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience,
|
||
imdb_id, tmdb_id, tvdb_id,
|
||
poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path,
|
||
metadata_s3_path, metadata_etag, season_count,
|
||
studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date,
|
||
matched_at, last_refreshed, refresh_failures, locked_fields, status,
|
||
created_at, updated_at
|
||
) VALUES (
|
||
$1, $2, $3, $4, $5, $6, $7,
|
||
$8, $9, $10, $11,
|
||
$12, $13, $14, $15,
|
||
$16, $17, $18,
|
||
$19, $20, $21, $22, $23,
|
||
$24, $25, $26,
|
||
$27, $28, $29, $30, $31, $32, $33, $34,
|
||
$35, $36, $37, $38, $39,
|
||
$40, $41
|
||
)
|
||
ON CONFLICT (content_id) DO UPDATE SET
|
||
type = EXCLUDED.type,
|
||
title = EXCLUDED.title,
|
||
sort_title = EXCLUDED.sort_title,
|
||
original_title = EXCLUDED.original_title,
|
||
year = EXCLUDED.year,
|
||
genres = EXCLUDED.genres,
|
||
content_rating = EXCLUDED.content_rating,
|
||
runtime = EXCLUDED.runtime,
|
||
overview = EXCLUDED.overview,
|
||
tagline = EXCLUDED.tagline,
|
||
rating_imdb = EXCLUDED.rating_imdb,
|
||
rating_tmdb = EXCLUDED.rating_tmdb,
|
||
rating_rt_critic = EXCLUDED.rating_rt_critic,
|
||
rating_rt_audience = EXCLUDED.rating_rt_audience,
|
||
imdb_id = EXCLUDED.imdb_id,
|
||
tmdb_id = EXCLUDED.tmdb_id,
|
||
tvdb_id = EXCLUDED.tvdb_id,
|
||
poster_path = EXCLUDED.poster_path,
|
||
poster_thumbhash = EXCLUDED.poster_thumbhash,
|
||
backdrop_path = EXCLUDED.backdrop_path,
|
||
backdrop_thumbhash = EXCLUDED.backdrop_thumbhash,
|
||
logo_path = EXCLUDED.logo_path,
|
||
metadata_s3_path = EXCLUDED.metadata_s3_path,
|
||
metadata_etag = EXCLUDED.metadata_etag,
|
||
season_count = EXCLUDED.season_count,
|
||
studios = EXCLUDED.studios,
|
||
networks = EXCLUDED.networks,
|
||
countries = EXCLUDED.countries,
|
||
keywords = EXCLUDED.keywords,
|
||
original_language = EXCLUDED.original_language,
|
||
release_date = EXCLUDED.release_date,
|
||
first_air_date = EXCLUDED.first_air_date,
|
||
last_air_date = EXCLUDED.last_air_date,
|
||
matched_at = EXCLUDED.matched_at,
|
||
last_refreshed = EXCLUDED.last_refreshed,
|
||
refresh_failures = EXCLUDED.refresh_failures,
|
||
locked_fields = EXCLUDED.locked_fields,
|
||
status = EXCLUDED.status,
|
||
updated_at = EXCLUDED.updated_at
|
||
RETURNING (xmax = 0)`,
|
||
item.ContentID, item.Type, item.Title, item.SortTitle, item.OriginalTitle, item.Year, item.Genres,
|
||
item.ContentRating, item.Runtime, item.Overview, item.Tagline,
|
||
item.RatingIMDB, item.RatingTMDB, item.RatingRTCritic, item.RatingRTAudience,
|
||
item.ImdbID, item.TmdbID, item.TvdbID,
|
||
item.PosterPath, item.PosterThumbhash, item.BackdropPath, item.BackdropThumbhash, item.LogoPath,
|
||
item.MetadataS3Path, item.MetadataEtag, item.SeasonCount,
|
||
item.Studios, item.Networks, item.Countries, item.Keywords, item.OriginalLanguage, item.ReleaseDate, item.FirstAirDate, item.LastAirDate,
|
||
item.MatchedAt, item.LastRefreshed, item.RefreshFailures, item.LockedFields, item.Status,
|
||
item.CreatedAt, item.UpdatedAt,
|
||
).Scan(&created)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("upserting item %s: %w", item.ContentID, err)
|
||
}
|
||
return created, !created, false, nil
|
||
}
|
||
|
||
func importSeason(ctx context.Context, tx pgx.Tx, season SeasonRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) {
|
||
if mode == ConflictModeSkipExisting {
|
||
tag, execErr := tx.Exec(ctx, `
|
||
INSERT INTO seasons (
|
||
content_id, series_id, season_number, title, overview, air_date,
|
||
poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||
ON CONFLICT (content_id) DO NOTHING`,
|
||
season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate,
|
||
season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag,
|
||
season.CreatedAt, season.UpdatedAt,
|
||
)
|
||
if execErr != nil {
|
||
return false, false, false, fmt.Errorf("importing season %s: %w", season.ContentID, execErr)
|
||
}
|
||
return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil
|
||
}
|
||
|
||
err = tx.QueryRow(ctx, `
|
||
INSERT INTO seasons (
|
||
content_id, series_id, season_number, title, overview, air_date,
|
||
poster_path, poster_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||
ON CONFLICT (content_id) DO UPDATE SET
|
||
series_id = EXCLUDED.series_id,
|
||
season_number = EXCLUDED.season_number,
|
||
title = EXCLUDED.title,
|
||
overview = EXCLUDED.overview,
|
||
air_date = EXCLUDED.air_date,
|
||
poster_path = EXCLUDED.poster_path,
|
||
poster_thumbhash = EXCLUDED.poster_thumbhash,
|
||
metadata_s3_path = EXCLUDED.metadata_s3_path,
|
||
metadata_etag = EXCLUDED.metadata_etag,
|
||
updated_at = EXCLUDED.updated_at
|
||
RETURNING (xmax = 0)`,
|
||
season.ContentID, season.SeriesID, season.SeasonNumber, season.Title, season.Overview, season.AirDate,
|
||
season.PosterPath, season.PosterThumbhash, season.MetadataS3Path, season.MetadataEtag,
|
||
season.CreatedAt, season.UpdatedAt,
|
||
).Scan(&created)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("upserting season %s: %w", season.ContentID, err)
|
||
}
|
||
return created, !created, false, nil
|
||
}
|
||
|
||
func importEpisode(ctx context.Context, tx pgx.Tx, episode EpisodeRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) {
|
||
seasonID := nullableString(episode.SeasonID)
|
||
if mode == ConflictModeSkipExisting {
|
||
tag, execErr := tx.Exec(ctx, `
|
||
INSERT INTO episodes (
|
||
content_id, series_id, season_id, season_number, episode_number,
|
||
title, overview, air_date, runtime, rating_imdb, rating_tmdb,
|
||
still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES (
|
||
$1, $2, $3, $4, $5,
|
||
$6, $7, $8, $9, $10, $11,
|
||
$12, $13, $14, $15, $16, $17
|
||
)
|
||
ON CONFLICT (content_id) DO NOTHING`,
|
||
episode.ContentID, episode.SeriesID, seasonID, episode.SeasonNumber, episode.EpisodeNumber,
|
||
episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB,
|
||
episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag,
|
||
episode.CreatedAt, episode.UpdatedAt,
|
||
)
|
||
if execErr != nil {
|
||
return false, false, false, fmt.Errorf("importing episode %s: %w", episode.ContentID, execErr)
|
||
}
|
||
return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil
|
||
}
|
||
|
||
err = tx.QueryRow(ctx, `
|
||
INSERT INTO episodes (
|
||
content_id, series_id, season_id, season_number, episode_number,
|
||
title, overview, air_date, runtime, rating_imdb, rating_tmdb,
|
||
still_path, still_thumbhash, metadata_s3_path, metadata_etag, created_at, updated_at
|
||
) VALUES (
|
||
$1, $2, $3, $4, $5,
|
||
$6, $7, $8, $9, $10, $11,
|
||
$12, $13, $14, $15, $16, $17
|
||
)
|
||
ON CONFLICT (content_id) DO UPDATE SET
|
||
series_id = EXCLUDED.series_id,
|
||
season_id = EXCLUDED.season_id,
|
||
season_number = EXCLUDED.season_number,
|
||
episode_number = EXCLUDED.episode_number,
|
||
title = EXCLUDED.title,
|
||
overview = EXCLUDED.overview,
|
||
air_date = EXCLUDED.air_date,
|
||
runtime = EXCLUDED.runtime,
|
||
rating_imdb = EXCLUDED.rating_imdb,
|
||
rating_tmdb = EXCLUDED.rating_tmdb,
|
||
still_path = EXCLUDED.still_path,
|
||
still_thumbhash = EXCLUDED.still_thumbhash,
|
||
metadata_s3_path = EXCLUDED.metadata_s3_path,
|
||
metadata_etag = EXCLUDED.metadata_etag,
|
||
updated_at = EXCLUDED.updated_at
|
||
RETURNING (xmax = 0)`,
|
||
episode.ContentID, episode.SeriesID, seasonID, episode.SeasonNumber, episode.EpisodeNumber,
|
||
episode.Title, episode.Overview, episode.AirDate, episode.Runtime, episode.RatingIMDB, episode.RatingTMDB,
|
||
episode.StillPath, episode.StillThumbhash, episode.MetadataS3Path, episode.MetadataEtag,
|
||
episode.CreatedAt, episode.UpdatedAt,
|
||
).Scan(&created)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("upserting episode %s: %w", episode.ContentID, err)
|
||
}
|
||
return created, !created, false, nil
|
||
}
|
||
|
||
func importFile(ctx context.Context, tx pgx.Tx, file FileRecord, mode ConflictMode) (created bool, updated bool, skipped bool, err error) {
|
||
videoTracksJSON, err := json.Marshal(file.VideoTracks)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("marshaling video tracks for %s: %w", file.FilePath, err)
|
||
}
|
||
audioTracksJSON, err := json.Marshal(file.AudioTracks)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("marshaling audio tracks for %s: %w", file.FilePath, err)
|
||
}
|
||
subtitleTracksJSON, err := json.Marshal(file.SubtitleTracks)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("marshaling subtitle tracks for %s: %w", file.FilePath, err)
|
||
}
|
||
externalSubtitlesJSON, err := json.Marshal(file.ExternalSubtitles)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("marshaling external subtitles for %s: %w", file.FilePath, err)
|
||
}
|
||
|
||
contentID := nullableString(file.ContentID)
|
||
episodeID := nullableString(file.EpisodeID)
|
||
fileHash := nullableString(file.FileHash)
|
||
probeSource := nullableString(file.ProbeSource)
|
||
if mode == ConflictModeSkipExisting {
|
||
tag, execErr := tx.Exec(ctx, `
|
||
INSERT INTO media_files (
|
||
content_id, episode_id, season_number, episode_number,
|
||
media_folder_id, file_path, file_size, file_hash,
|
||
codec_video, codec_audio, resolution, audio_channels, hdr, container,
|
||
duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles,
|
||
intro_start, intro_end, credits_start, credits_end,
|
||
probe_source, probe_updated_at, missing_since, created_at, updated_at
|
||
) VALUES (
|
||
$1, $2, $3, $4,
|
||
$5, $6, $7, $8,
|
||
$9, $10, $11, $12, $13, $14,
|
||
$15, $16, $17, $18, $19, $20,
|
||
$21, $22, $23, $24,
|
||
$25, $26, $27, $28, $29
|
||
)
|
||
ON CONFLICT (file_path) DO NOTHING`,
|
||
contentID, episodeID, nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber),
|
||
file.MediaFolderID, file.FilePath, file.FileSize, fileHash,
|
||
file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container,
|
||
nilIfZero(file.Duration), nilIfZero(file.Bitrate), videoTracksJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON,
|
||
file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd,
|
||
probeSource, file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt,
|
||
)
|
||
if execErr != nil {
|
||
return false, false, false, fmt.Errorf("importing file %s: %w", file.FilePath, execErr)
|
||
}
|
||
return tag.RowsAffected() == 1, false, tag.RowsAffected() == 0, nil
|
||
}
|
||
|
||
err = tx.QueryRow(ctx, `
|
||
INSERT INTO media_files (
|
||
content_id, episode_id, season_number, episode_number,
|
||
media_folder_id, file_path, file_size, file_hash,
|
||
codec_video, codec_audio, resolution, audio_channels, hdr, container,
|
||
duration, bitrate, video_tracks, audio_tracks, subtitle_tracks, external_subtitles,
|
||
intro_start, intro_end, credits_start, credits_end,
|
||
probe_source, probe_updated_at, missing_since, created_at, updated_at
|
||
) VALUES (
|
||
$1, $2, $3, $4,
|
||
$5, $6, $7, $8,
|
||
$9, $10, $11, $12, $13, $14,
|
||
$15, $16, $17, $18, $19, $20,
|
||
$21, $22, $23, $24,
|
||
$25, $26, $27, $28, $29
|
||
)
|
||
ON CONFLICT (file_path) DO UPDATE SET
|
||
content_id = EXCLUDED.content_id,
|
||
episode_id = EXCLUDED.episode_id,
|
||
season_number = EXCLUDED.season_number,
|
||
episode_number = EXCLUDED.episode_number,
|
||
media_folder_id = EXCLUDED.media_folder_id,
|
||
file_size = EXCLUDED.file_size,
|
||
file_hash = EXCLUDED.file_hash,
|
||
codec_video = EXCLUDED.codec_video,
|
||
codec_audio = EXCLUDED.codec_audio,
|
||
resolution = EXCLUDED.resolution,
|
||
audio_channels = EXCLUDED.audio_channels,
|
||
hdr = EXCLUDED.hdr,
|
||
container = EXCLUDED.container,
|
||
duration = EXCLUDED.duration,
|
||
bitrate = EXCLUDED.bitrate,
|
||
video_tracks = EXCLUDED.video_tracks,
|
||
audio_tracks = EXCLUDED.audio_tracks,
|
||
subtitle_tracks = EXCLUDED.subtitle_tracks,
|
||
external_subtitles = EXCLUDED.external_subtitles,
|
||
intro_start = EXCLUDED.intro_start,
|
||
intro_end = EXCLUDED.intro_end,
|
||
credits_start = EXCLUDED.credits_start,
|
||
credits_end = EXCLUDED.credits_end,
|
||
probe_source = EXCLUDED.probe_source,
|
||
probe_updated_at = EXCLUDED.probe_updated_at,
|
||
missing_since = EXCLUDED.missing_since,
|
||
updated_at = EXCLUDED.updated_at
|
||
RETURNING (xmax = 0)`,
|
||
contentID, episodeID, nilIfZero(file.SeasonNumber), nilIfZero(file.EpisodeNumber),
|
||
file.MediaFolderID, file.FilePath, file.FileSize, fileHash,
|
||
file.CodecVideo, file.CodecAudio, file.Resolution, nilIfZero(file.AudioChannels), file.HDR, file.Container,
|
||
nilIfZero(file.Duration), nilIfZero(file.Bitrate), videoTracksJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON,
|
||
file.IntroStart, file.IntroEnd, file.CreditsStart, file.CreditsEnd,
|
||
probeSource, file.ProbeUpdatedAt, file.MissingSince, file.CreatedAt, file.UpdatedAt,
|
||
).Scan(&created)
|
||
if err != nil {
|
||
return false, false, false, fmt.Errorf("upserting file %s: %w", file.FilePath, err)
|
||
}
|
||
return created, !created, false, nil
|
||
}
|
||
|
||
func importLibraryLink(ctx context.Context, tx pgx.Tx, link LibraryLinkRecord) (bool, error) {
|
||
tag, err := tx.Exec(ctx, `
|
||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||
VALUES ($1, $2, $3)
|
||
ON CONFLICT (content_id, media_folder_id) DO NOTHING`,
|
||
link.ContentID, link.MediaFolderID, link.FirstSeenAt,
|
||
)
|
||
if err != nil {
|
||
return false, fmt.Errorf("importing library link %s:%d: %w", link.ContentID, link.MediaFolderID, err)
|
||
}
|
||
return tag.RowsAffected() == 1, nil
|
||
}
|
||
|
||
func normalizeRewrites(rewrites []PathRewrite) []PathRewrite {
|
||
out := make([]PathRewrite, 0, len(rewrites))
|
||
for _, rewrite := range rewrites {
|
||
from := normalizePath(rewrite.From)
|
||
to := normalizePath(rewrite.To)
|
||
if from == "" || to == "" {
|
||
continue
|
||
}
|
||
out = append(out, PathRewrite{From: from, To: to})
|
||
}
|
||
sort.Slice(out, func(i, j int) bool { return len(out[i].From) > len(out[j].From) })
|
||
return out
|
||
}
|
||
|
||
func rewritePath(path string, rewrites []PathRewrite) (string, bool) {
|
||
normalized := normalizePath(path)
|
||
for _, rewrite := range rewrites {
|
||
if !hasPathPrefix(normalized, rewrite.From) {
|
||
continue
|
||
}
|
||
if normalized == rewrite.From {
|
||
return rewrite.To, true
|
||
}
|
||
suffix := strings.TrimPrefix(normalized, rewrite.From)
|
||
return normalizePath(rewrite.To + suffix), true
|
||
}
|
||
return normalized, false
|
||
}
|
||
|
||
func normalizePath(path string) string {
|
||
if path == "" {
|
||
return ""
|
||
}
|
||
return filepath.Clean(path)
|
||
}
|
||
|
||
func hasPathPrefix(path, prefix string) bool {
|
||
path = normalizePath(path)
|
||
prefix = normalizePath(prefix)
|
||
if path == prefix {
|
||
return true
|
||
}
|
||
if prefix == string(os.PathSeparator) {
|
||
return strings.HasPrefix(path, prefix)
|
||
}
|
||
return strings.HasPrefix(path, prefix+string(os.PathSeparator))
|
||
}
|
||
|
||
func sortLibraries(records []LibraryRecord) {
|
||
sort.Slice(records, func(i, j int) bool { return records[i].ExportedID < records[j].ExportedID })
|
||
}
|
||
|
||
func cloneStrings(values []string) []string {
|
||
if values == nil {
|
||
return []string{}
|
||
}
|
||
return append([]string(nil), values...)
|
||
}
|
||
|
||
func itemRecordStringArrays(item ItemRecord) ([]string, []string, []string, []string) {
|
||
return nonNilStrings(item.Studios),
|
||
nonNilStrings(item.Networks),
|
||
nonNilStrings(item.Countries),
|
||
nonNilStrings(item.Keywords)
|
||
}
|
||
|
||
func nonNilStrings(values []string) []string {
|
||
if values == nil {
|
||
return []string{}
|
||
}
|
||
return values
|
||
}
|
||
|
||
func cloneInts(values []int) []int {
|
||
if values == nil {
|
||
return []int{}
|
||
}
|
||
return append([]int(nil), values...)
|
||
}
|
||
|
||
func dedupeStrings(values []string) []string {
|
||
if len(values) == 0 {
|
||
return []string{}
|
||
}
|
||
seen := make(map[string]struct{}, len(values))
|
||
out := make([]string, 0, len(values))
|
||
for _, value := range values {
|
||
if _, ok := seen[value]; ok {
|
||
continue
|
||
}
|
||
seen[value] = struct{}{}
|
||
out = append(out, value)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func mapKeys(m map[string]struct{}) []string {
|
||
keys := make([]string, 0, len(m))
|
||
for key := range m {
|
||
keys = append(keys, key)
|
||
}
|
||
return keys
|
||
}
|
||
|
||
func nilIfZero(value int) any {
|
||
if value == 0 {
|
||
return nil
|
||
}
|
||
return value
|
||
}
|
||
|
||
func nullableString(value string) any {
|
||
if value == "" {
|
||
return nil
|
||
}
|
||
return value
|
||
}
|
||
|
||
func toVideoTrackRecords(tracks []models.VideoTrack) []VideoTrackRecord {
|
||
out := make([]VideoTrackRecord, 0, len(tracks))
|
||
for _, track := range tracks {
|
||
out = append(out, VideoTrackRecord{
|
||
Title: track.Title,
|
||
Codec: track.Codec,
|
||
DolbyVision: track.DolbyVision,
|
||
Profile: track.Profile,
|
||
Level: track.Level,
|
||
Width: track.Width,
|
||
Height: track.Height,
|
||
AspectRatio: track.AspectRatio,
|
||
Interlaced: track.Interlaced,
|
||
FrameRate: track.FrameRate,
|
||
Bitrate: track.Bitrate,
|
||
VideoRange: track.VideoRange,
|
||
ColorPrimaries: track.ColorPrimaries,
|
||
ColorSpace: track.ColorSpace,
|
||
ColorTransfer: track.ColorTransfer,
|
||
BitDepth: track.BitDepth,
|
||
PixelFormat: track.PixelFormat,
|
||
ReferenceFrames: track.ReferenceFrames,
|
||
})
|
||
}
|
||
return out
|
||
}
|
||
|
||
func toAudioTrackRecords(tracks []models.AudioTrack) []AudioTrackRecord {
|
||
out := make([]AudioTrackRecord, 0, len(tracks))
|
||
for _, track := range tracks {
|
||
out = append(out, AudioTrackRecord{
|
||
Title: track.Title,
|
||
EmbeddedTitle: track.EmbeddedTitle,
|
||
Language: track.Language,
|
||
Codec: track.Codec,
|
||
Layout: track.Layout,
|
||
Channels: track.Channels,
|
||
Bitrate: track.Bitrate,
|
||
SampleRate: track.SampleRate,
|
||
BitDepth: track.BitDepth,
|
||
Default: track.Default,
|
||
})
|
||
}
|
||
return out
|
||
}
|
||
|
||
func toSubtitleTrackRecords(tracks []models.SubtitleTrack) []SubtitleTrackRecord {
|
||
out := make([]SubtitleTrackRecord, 0, len(tracks))
|
||
for _, track := range tracks {
|
||
out = append(out, SubtitleTrackRecord{
|
||
Index: track.Index,
|
||
Language: track.Language,
|
||
Codec: track.Codec,
|
||
Title: track.Title,
|
||
EmbeddedTitle: track.EmbeddedTitle,
|
||
Resolution: track.Resolution,
|
||
Forced: track.Forced,
|
||
Default: track.Default,
|
||
HearingImpaired: track.HearingImpaired,
|
||
External: track.External,
|
||
FileName: track.FileName,
|
||
})
|
||
}
|
||
return out
|
||
}
|
||
|
||
func toExternalSubtitleRecords(tracks []models.ExternalSubtitle) []ExternalSubtitleRecord {
|
||
out := make([]ExternalSubtitleRecord, 0, len(tracks))
|
||
for _, track := range tracks {
|
||
out = append(out, ExternalSubtitleRecord{
|
||
Path: track.Path,
|
||
Language: track.Language,
|
||
Format: track.Format,
|
||
Title: track.Title,
|
||
EmbeddedTitle: track.EmbeddedTitle,
|
||
Resolution: track.Resolution,
|
||
Forced: track.Forced,
|
||
Default: track.Default,
|
||
HearingImpaired: track.HearingImpaired,
|
||
})
|
||
}
|
||
return out
|
||
}
|