diff --git a/internal/scanner/extras.go b/internal/scanner/extras.go index 5062ad5a..a89b66f7 100644 --- a/internal/scanner/extras.go +++ b/internal/scanner/extras.go @@ -31,20 +31,73 @@ type extraCandidate struct { // live inside a directory named "Extras" classify everything beneath it. const extrasDirAncestorDepth = 2 -// classifyExtraPath reports whether the walked path is a local extra. +// extrasClassifier classifies walked paths as local extras using the +// library's structure. A convention-named directory ("Other", "Trailers", +// "Extras", ...) only counts as an extras dir when it is owned by a title +// folder — a directory that holds media of its own (the movie file beside the +// extras dir, or episodes in season folders beside it). Convention names used +// as content-scope folders at any depth ("movies/other//...", +// "movies/4K/shorts//...") own no media directly and never classify, +// so the titles beneath them stay primary. +type extrasClassifier struct { + folderType string + rootSet map[string]bool + // dirFiles marks directories that directly contain a walked media file. + dirFiles map[string]bool + // dirFilesBelow marks directories with a walked media file exactly two + // levels down through a non-convention child (a show folder above its + // season folders — but not a folder whose only media hides inside its + // own extras dirs). + dirFilesBelow map[string]bool + // probeFS switches ownership checks to bounded os.ReadDir probes for + // single-file (watch event) scans, which have no walked path list. + probeFS bool +} + +// newExtrasClassifier builds a classifier from a scan's walked paths. +func newExtrasClassifier(folderType string, libraryRoots []string, walkedPaths []string) *extrasClassifier { + c := &extrasClassifier{ + folderType: folderType, + rootSet: walkRootSet(libraryRoots), + dirFiles: make(map[string]bool, len(walkedPaths)), + dirFilesBelow: make(map[string]bool, len(walkedPaths)), + } + for _, p := range walkedPaths { + dir := filepath.Dir(p) + c.dirFiles[dir] = true + if extrasDirKinds[normalizeScannerDirLabel(filepath.Base(dir))] == "" { + c.dirFilesBelow[filepath.Dir(dir)] = true + } + } + return c +} + +// newWatchExtrasClassifier builds a classifier for single-file scans; title +// ownership is probed from the filesystem instead of a walked path list. +func newWatchExtrasClassifier(folderType string, libraryRoots []string) *extrasClassifier { + return &extrasClassifier{ + folderType: folderType, + rootSet: walkRootSet(libraryRoots), + probeFS: true, + } +} + +// classify reports whether the walked path is a local extra. // // Directory names win over filename suffixes. For non-movie libraries a file // carrying a parseable SxxExx episode token is never an extra: series // "Extras/SxxExx" files keep their documented season-0 mapping. -func classifyExtraPath(path, folderType string) (extraCandidate, bool) { +func (c *extrasClassifier) classify(path string) (extraCandidate, bool) { candidate := extraCandidate{Path: path} dir := filepath.Dir(path) for depth := 0; depth < extrasDirAncestorDepth; depth++ { label := normalizeScannerDirLabel(filepath.Base(dir)) if kind, ok := extrasDirKinds[label]; ok { - candidate.Kind = kind - candidate.SupplementalDir = dir + if c.titleDirOwns(dir) { + candidate.Kind = kind + candidate.SupplementalDir = dir + } break } parent := filepath.Dir(dir) @@ -64,8 +117,8 @@ func classifyExtraPath(path, folderType string) (extraCandidate, bool) { // Preserve the documented series behavior: an episode-tokened file under // Extras/ is a season-0 special, not an extra. - if !librarykind.IsMovie(folderType) { - if hints := naming.ParseFilename(path, folderType); hints != nil && + if !librarykind.IsMovie(c.folderType) { + if hints := naming.ParseFilename(path, c.folderType); hints != nil && hints.Type == "series" && hints.EpisodeNum > 0 { return extraCandidate{}, false } @@ -74,14 +127,86 @@ func classifyExtraPath(path, folderType string) (extraCandidate, bool) { return candidate, true } +// titleDirOwns reports whether the matched supplemental directory is owned by +// a title folder: the first non-supplemental ancestor must not be a library +// root and must hold media of its own — directly for movie folders, or one +// level down for series folders whose episodes live in season subfolders. +func (c *extrasClassifier) titleDirOwns(supplementalDir string) bool { + owner := firstNonSupplementalAncestor(supplementalDir) + if c.rootSet[owner] { + return false + } + if c.probeFS { + depth := 1 + if !librarykind.IsMovie(c.folderType) { + depth = 2 + } + return c.dirHoldsMedia(owner, depth) + } + if c.dirFiles[owner] { + return true + } + return !librarykind.IsMovie(c.folderType) && c.dirFilesBelow[owner] +} + +// dirHoldsMedia is the probeFS counterpart of dirFiles/dirFilesBelow: it +// reports whether dir holds a media file within depth levels, without +// descending into convention-named subdirectories. +func (c *extrasClassifier) dirHoldsMedia(dir string, depth int) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + mode := walkModeFor(c.folderType) + for _, entry := range entries { + if entry.IsDir() { + if depth > 1 && extrasDirKinds[normalizeScannerDirLabel(entry.Name())] == "" && + c.dirHoldsMedia(filepath.Join(dir, entry.Name()), depth-1) { + return true + } + continue + } + if mode.acceptsExt(strings.ToLower(filepath.Ext(entry.Name()))) { + return true + } + } + return false +} + +// firstNonSupplementalAncestor walks up from a supplemental directory past any +// chained convention names ("Extras/Behind The Scenes/") and returns the +// cleaned directory that owns the supplemental chain. +func firstNonSupplementalAncestor(supplementalDir string) string { + dir := filepath.Dir(filepath.Clean(supplementalDir)) + for extrasDirKinds[normalizeScannerDirLabel(filepath.Base(dir))] != "" { + next := filepath.Dir(dir) + if next == dir { + break + } + dir = next + } + return dir +} + +// walkRootSet builds the cleaned-path set used for scope checks against the +// library's configured roots. +func walkRootSet(roots []string) map[string]bool { + set := make(map[string]bool, len(roots)) + for _, root := range roots { + set[filepath.Clean(root)] = true + } + return set +} + // partitionExtraPaths splits walked paths into primary content and extras. // Primary paths feed the existing root/group inference and matching pipeline // untouched; extras are processed separately and never influence identity. -func partitionExtraPaths(paths []string, folderType string) ([]string, []extraCandidate) { +func partitionExtraPaths(paths []string, folderType string, libraryRoots []string) ([]string, []extraCandidate) { + classifier := newExtrasClassifier(folderType, libraryRoots, paths) primary := paths[:0:0] var extras []extraCandidate for _, p := range paths { - if candidate, ok := classifyExtraPath(p, folderType); ok { + if candidate, ok := classifier.classify(p); ok { extras = append(extras, candidate) continue } @@ -107,7 +232,6 @@ type extrasScanStats struct { func (s *Scanner) processExtraFiles( ctx context.Context, folder *models.MediaFolder, - walkRoots []string, extras []extraCandidate, existingByPath map[string]*scanStateFile, ) extrasScanStats { @@ -116,10 +240,10 @@ func (s *Scanner) processExtraFiles( return stats } - rootSet := make(map[string]bool, len(walkRoots)) - for _, root := range walkRoots { - rootSet[filepath.Clean(root)] = true - } + // Parent binding is scoped by the library's configured roots, not the + // (possibly narrower) walk roots of a scoped scan: a movie folder targeted + // directly by a subtree scan must still bind its own extras. + rootSet := walkRootSet(folder.Paths) for _, candidate := range extras { if ctx.Err() != nil { @@ -256,16 +380,7 @@ func (s *Scanner) resolveExtraParent( return s.fileRepo.FindUnambiguousParentContentIDForDir(ctx, folderID, dir) } - parentDir := filepath.Dir(candidate.SupplementalDir) - // Walk supplemental nesting ("Extras/Behind The Scenes/") up to the first - // non-supplemental ancestor. - for extrasDirKinds[normalizeScannerDirLabel(filepath.Base(parentDir))] != "" { - next := filepath.Dir(parentDir) - if next == parentDir { - break - } - parentDir = next - } + parentDir := firstNonSupplementalAncestor(candidate.SupplementalDir) if rootSet[filepath.Clean(parentDir)] { // Supplemental dir sits at the library root — no single owner. return "", nil diff --git a/internal/scanner/extras_test.go b/internal/scanner/extras_test.go index 4702a8ef..b7683b72 100644 --- a/internal/scanner/extras_test.go +++ b/internal/scanner/extras_test.go @@ -1,6 +1,8 @@ package scanner import ( + "os" + "path/filepath" "testing" "github.com/Silo-Server/silo-server/internal/models" @@ -16,42 +18,113 @@ func TestClassifyExtraPathMovieLibrary(t *testing.T) { {"/movies/Heat (1995)/Trailers/teaser.mkv", models.ExtraKindTrailer, "/movies/Heat (1995)/Trailers", true}, {"/movies/Heat (1995)/Behind The Scenes/doc.mkv", models.ExtraKindBehindTheScenes, "/movies/Heat (1995)/Behind The Scenes", true}, {"/movies/Heat (1995)/Extras/Making Of.mkv", models.ExtraKindOther, "/movies/Heat (1995)/Extras", true}, + // "Other" is part of the Jellyfin/Plex extras convention. + {"/movies/Heat (1995)/Other/making-of.mkv", models.ExtraKindOther, "/movies/Heat (1995)/Other", true}, // Nested one level below a supplemental dir still classifies. {"/movies/Heat (1995)/Extras/Sub/clip.mkv", models.ExtraKindOther, "/movies/Heat (1995)/Extras", true}, + // Title folders own their extras at any depth below the root. + {"/movies/Collection/Ronin (1998)/Other/interview.mkv", models.ExtraKindOther, "/movies/Collection/Ronin (1998)/Other", true}, // Suffix classification with no supplemental dir. {"/movies/Heat (1995)/Heat (1995)-trailer.mkv", models.ExtraKindTrailer, "", true}, // Plain movie files are not extras. {"/movies/Heat (1995)/Heat (1995).mkv", "", "", false}, + {"/movies/Collection/Ronin (1998)/Ronin (1998).mkv", "", "", false}, // Ancestor lookup is depth-bounded: a library living under a dir // named "Extras" must not classify everything. {"/data/Extras/Movies/Heat (1995)/Heat (1995).mkv", "", "", false}, + // A content-scope folder carrying a convention label ("other", + // "shorts", "extras", ...) owns no media of its own, so titles + // beneath it stay primary and must not be misclassified as extras + // (regression for the /movies/other re-probe/defer storm) — at the + // library root or nested any depth below it. "others" is additionally + // absent from the convention vocabulary entirely. + {"/movies/other/Heat (1995)/Heat (1995).mkv", "", "", false}, + {"/movies/others/Heat (1995)/Heat (1995).mkv", "", "", false}, + {"/movies/shorts/Heat (1995)/Heat (1995).mkv", "", "", false}, + {"/movies/4K/other/Alien (1979)/Alien (1979).mkv", "", "", false}, + // Chained convention names at library scope hold no title either: + // loose clips there stay primary instead of deferring forever. + {"/movies/extras/behind the scenes/clip.mkv", "", "", false}, + // Loose files directly under a scope-level convention dir are primary + // too — unless the filename itself carries a convention suffix. + {"/movies/other/stray file.mkv", "", "", false}, } + paths := make([]string, 0, len(cases)) for _, tc := range cases { - candidate, ok := classifyExtraPath(tc.path, "movies") + paths = append(paths, tc.path) + } + classifier := newExtrasClassifier("movies", []string{"/movies"}, paths) + for _, tc := range cases { + candidate, ok := classifier.classify(tc.path) if ok != tc.wantOK { - t.Errorf("classifyExtraPath(%q) ok = %v, want %v", tc.path, ok, tc.wantOK) + t.Errorf("classify(%q) ok = %v, want %v", tc.path, ok, tc.wantOK) continue } if !ok { continue } if candidate.Kind != tc.wantKind || candidate.SupplementalDir != tc.wantDir { - t.Errorf("classifyExtraPath(%q) = (%q, %q), want (%q, %q)", + t.Errorf("classify(%q) = (%q, %q), want (%q, %q)", tc.path, candidate.Kind, candidate.SupplementalDir, tc.wantKind, tc.wantDir) } } } -func TestClassifyExtraPathSeriesKeepsSeasonZeroBehavior(t *testing.T) { +func TestClassifyExtraPathSeriesLibrary(t *testing.T) { + paths := []string{ + "/tv/Show/Season 01/Show S01E01.mkv", + "/tv/Show/Extras/Show S00E01 Special.mkv", + "/tv/Show/Trailers/season-preview.mkv", + "/tv/other/Flat Show/pilot.mkv", + } + classifier := newExtrasClassifier("series", []string{"/tv"}, paths) + // Documented behavior: an episode-tokened file under Extras/ in a series // library maps to season 0, so it must NOT classify as an extra. - if _, ok := classifyExtraPath("/tv/Show/Extras/Show S00E01 Special.mkv", "series"); ok { + if _, ok := classifier.classify("/tv/Show/Extras/Show S00E01 Special.mkv"); ok { t.Fatal("SxxExx file under Extras/ must remain a season-0 episode, not an extra") } - // A non-tokened file under a series-root supplemental dir IS an extra. - candidate, ok := classifyExtraPath("/tv/Show/Trailers/season-preview.mkv", "series") + // A non-tokened file under a show-level supplemental dir IS an extra; + // the show folder owns it through its season-level episodes. + candidate, ok := classifier.classify("/tv/Show/Trailers/season-preview.mkv") if !ok || candidate.Kind != models.ExtraKindTrailer { - t.Fatalf("series-root trailer dir should classify, got ok=%v kind=%q", ok, candidate.Kind) + t.Fatalf("show trailer dir should classify, got ok=%v kind=%q", ok, candidate.Kind) + } + // A scope folder named "other" holding show folders stays primary. + if _, ok := classifier.classify("/tv/other/Flat Show/pilot.mkv"); ok { + t.Fatal("show under a scope-level other/ must remain primary") + } +} + +func TestClassifyExtraPathWatchMode(t *testing.T) { + // Watch-event scans have no walked path list; title ownership is probed + // from the filesystem. + root := t.TempDir() + title := filepath.Join(root, "Heat (1995)") + other := filepath.Join(title, "Other") + scopeOther := filepath.Join(root, "other", "Alien (1979)") + for _, dir := range []string{other, scopeOther} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + for _, file := range []string{ + filepath.Join(title, "Heat (1995).mkv"), + filepath.Join(other, "making-of.mkv"), + filepath.Join(scopeOther, "Alien (1979).mkv"), + } { + if err := os.WriteFile(file, nil, 0o644); err != nil { + t.Fatal(err) + } + } + + classifier := newWatchExtrasClassifier("movies", []string{root}) + candidate, ok := classifier.classify(filepath.Join(other, "making-of.mkv")) + if !ok || candidate.Kind != models.ExtraKindOther { + t.Fatalf("convention dir beside the movie file should classify, got ok=%v kind=%q", ok, candidate.Kind) + } + if _, ok := classifier.classify(filepath.Join(scopeOther, "Alien (1979).mkv")); ok { + t.Fatal("title under a scope-level other/ must remain primary in watch mode") } } @@ -61,7 +134,7 @@ func TestPartitionExtraPaths(t *testing.T) { "/movies/Heat (1995)/Trailers/tease.mkv", "/movies/Heat (1995)/Heat (1995)-featurette.mkv", } - primary, extras := partitionExtraPaths(paths, "movies") + primary, extras := partitionExtraPaths(paths, "movies", []string{"/movies"}) if len(primary) != 1 || primary[0] != paths[0] { t.Fatalf("primary = %v, want just the main feature", primary) } diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index 3099343e..32ff40b1 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -842,22 +842,7 @@ func (r *FileRepository) Upsert(ctx context.Context, mf models.MediaFile) (*mode if mf.ProbeSource != "" { probeSource = &mf.ProbeSource } - var editionConfidence *float64 - if mf.EditionConfidence != nil { - editionConfidence = mf.EditionConfidence - } - groupKeyVersion := mf.GroupKeyVersion - if groupKeyVersion == 0 { - groupKeyVersion = 1 - } - identityConfidence := mf.IdentityConfidence - if identityConfidence == "" { - identityConfidence = "low" - } - identityJSON := mf.IdentityJSON - if len(identityJSON) == 0 { - identityJSON = []byte("{}") - } + groupKeyVersion, identityConfidence, identityJSON := identityColumnDefaults(mf) query := `INSERT INTO media_files ( content_id, episode_id, extra_id, season_number, episode_number, @@ -986,7 +971,7 @@ func (r *FileRepository) Upsert(ctx context.Context, mf models.MediaFile) (*mode mf.MarkersConfidence, mf.EditionRaw, mf.EditionKey, - editionConfidence, + mf.EditionConfidence, mf.EditionSource, mf.PresentationKind, mf.PresentationGroupKey, @@ -1002,6 +987,105 @@ func (r *FileRepository) Upsert(ctx context.Context, mf models.MediaFile) (*mode return scanMediaFile(row) } +// identityColumnDefaults normalizes the identity/grouping zero values the way +// every media_files write must persist them. Upsert and UpdateIdentity both go +// through it so the full and metadata-only scan paths converge on identical +// stored values. +func identityColumnDefaults(mf models.MediaFile) (groupKeyVersion int, identityConfidence string, identityJSON []byte) { + groupKeyVersion = mf.GroupKeyVersion + if groupKeyVersion == 0 { + groupKeyVersion = 1 + } + identityConfidence = mf.IdentityConfidence + if identityConfidence == "" { + identityConfidence = "low" + } + identityJSON = mf.IdentityJSON + if len(identityJSON) == 0 { + identityJSON = []byte("{}") + } + return groupKeyVersion, identityConfidence, identityJSON +} + +// UpdateIdentity rewrites only the derived root/group/identity and +// edition/presentation columns of an existing media_files row, returning the +// row id. Probe data, file bytes/mtime/hash, subtitles, chapters, markers, and +// content/episode/extra linkage are left untouched. It backs the scanner's +// metadata-only update path: an identity or content-group-key reclassification +// must persist the new grouping without re-running (or disturbing) ffprobe. +// Column handling mirrors Upsert's ON CONFLICT assignments for the same +// columns so the two paths converge on identical values; like any scan write, +// it clears match suppression so the fresh identity re-enters the match +// backlog. Only the id is returned — this runs once per file during +// library-wide grouping migrations, and returning the full row would drag the +// track/chapter JSONB payloads along for millions of rows. Returns +// ErrFileNotFound when the row no longer exists. +func (r *FileRepository) UpdateIdentity(ctx context.Context, mf models.MediaFile) (int, error) { + groupKeyVersion, identityConfidence, identityJSON := identityColumnDefaults(mf) + + query := `UPDATE media_files SET + media_folder_id = $2, + canonical_root_path = $3, + observed_root_path = $4, + content_group_key = $5, + group_key_version = $6, + base_title = $7, + base_year = $8, + base_type = $9, + identity_confidence = $10, + identity_json = $11, + season_number = COALESCE($12, season_number), + episode_number = COALESCE($13, episode_number), + edition_raw = $14, + edition_key = $15, + edition_confidence = $16, + edition_source = $17, + presentation_kind = $18, + presentation_group_key = $19, + presentation_part_index = $20, + presentation_part_total = $21, + multi_episode_start = $22, + multi_episode_end = $23, + match_suppressed_at = NULL, + updated_at = NOW() + WHERE file_path = $1 + RETURNING id` + + var id int + err := r.pool.QueryRow(ctx, query, + mf.FilePath, + mf.MediaFolderID, + mf.CanonicalRootPath, + mf.ObservedRootPath, + mf.ContentGroupKey, + groupKeyVersion, + mf.BaseTitle, + mf.BaseYear, + mf.BaseType, + identityConfidence, + identityJSON, + nilIfZero(mf.SeasonNumber), + nilIfZero(mf.EpisodeNumber), + mf.EditionRaw, + mf.EditionKey, + mf.EditionConfidence, + mf.EditionSource, + mf.PresentationKind, + mf.PresentationGroupKey, + nilIfZero(mf.PresentationPartIndex), + nilIfZero(mf.PresentationPartTotal), + nilIfZero(mf.MultiEpisodeStart), + nilIfZero(mf.MultiEpisodeEnd), + ).Scan(&id) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return 0, ErrFileNotFound + } + return 0, fmt.Errorf("updating media file identity: %w", err) + } + return id, nil +} + type ChapterThumbnailFailureState struct { Apply bool RetryAfter *time.Time diff --git a/internal/scanner/scan_state.go b/internal/scanner/scan_state.go index 747fd180..47e7442c 100644 --- a/internal/scanner/scan_state.go +++ b/internal/scanner/scan_state.go @@ -28,6 +28,7 @@ type scanStateFile struct { FilePath string FileSize int64 FileModifiedAt *time.Time + FileHash string CodecVideo string CodecAudio string Resolution string @@ -54,7 +55,7 @@ type scanStateFile struct { const scanStateColumns = `id, content_id, extra_id, canonical_root_path, observed_root_path, content_group_key, group_key_version, base_title, base_year, base_type, identity_confidence, identity_json, - file_path, file_size, file_modified_at, + file_path, file_size, file_modified_at, file_hash, codec_video, codec_audio, resolution, container, duration, edition_raw, edition_key, edition_confidence, edition_source, presentation_kind, presentation_group_key, presentation_part_index, @@ -81,6 +82,7 @@ func scanScanStateRow(row pgx.Row) (*scanStateFile, error) { var identityConfidence *string var identityJSON []byte var fileModifiedAt *time.Time + var fileHash *string var codecVideo, codecAudio, resolution, container *string var duration *int var editionRaw, editionKey, editionSource *string @@ -105,6 +107,7 @@ func scanScanStateRow(row pgx.Row) (*scanStateFile, error) { &state.FilePath, &state.FileSize, &fileModifiedAt, + &fileHash, &codecVideo, &codecAudio, &resolution, @@ -164,6 +167,9 @@ func scanScanStateRow(row pgx.Row) (*scanStateFile, error) { state.IdentityJSON = append([]byte(nil), identityJSON...) } state.FileModifiedAt = fileModifiedAt + if fileHash != nil { + state.FileHash = *fileHash + } if codecVideo != nil { state.CodecVideo = *codecVideo } @@ -276,6 +282,7 @@ func scanStateFromMediaFile(file *models.MediaFile) *scanStateFile { FilePath: file.FilePath, FileSize: file.FileSize, FileModifiedAt: file.FileModifiedAt, + FileHash: file.FileHash, CodecVideo: file.CodecVideo, CodecAudio: file.CodecAudio, Resolution: file.Resolution, diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index d59e14a2..55945f11 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -63,12 +63,19 @@ var ignoredMovieSupplementalDirNames = map[string]bool{ // extrasDirKinds classifies supplemental directory names (normalized via // normalizeScannerDirLabel) into the shared extra-kind vocabulary. The set -// mirrors the Jellyfin/Plex extras folder convention. +// mirrors the Jellyfin/Plex extras folder convention ("other" included: both +// conventions document it). +// +// Deliberately absent: the plural "others", which is in neither convention. +// Convention labels can also appear as content-scope folder names ("movies/ +// other//", "movies/shorts/..."); those never classify as extras +// because extrasClassifier only honors a convention-named dir owned by a +// title folder — one that holds media of its own, which library roots and +// organizational folders do not. var extrasDirKinds = map[string]models.ExtraKind{ "extra": models.ExtraKindOther, "extras": models.ExtraKindOther, "other": models.ExtraKindOther, - "others": models.ExtraKindOther, "featurette": models.ExtraKindFeaturette, "featurettes": models.ExtraKindFeaturette, "behind the scenes": models.ExtraKindBehindTheScenes, @@ -687,7 +694,7 @@ func (s *Scanner) scanPaths( for _, p := range filePaths { seenPaths[p] = true } - primaryPaths, extraCandidates := partitionExtraPaths(filePaths, folder.Type) + primaryPaths, extraCandidates := partitionExtraPaths(filePaths, folder.Type, folder.Paths) rootOverrides, err := s.loadRootOverrides(ctx, folder.ID, reconcileRoots) if err != nil { return nil, fmt.Errorf("loading root overrides: %w", err) @@ -845,7 +852,7 @@ func (s *Scanner) scanPaths( return result, ctx.Err() } - extraStats := s.processExtraFiles(ctx, folder, walkRoots, extraCandidates, existingByPath) + extraStats := s.processExtraFiles(ctx, folder, extraCandidates, existingByPath) result.New += extraStats.New result.Updated += extraStats.Updated result.Unchanged += extraStats.Unchanged @@ -1207,7 +1214,7 @@ func (s *Scanner) scanScope( for _, p := range filePaths { seenPaths[p] = true } - primaryPaths, extraCandidates := partitionExtraPaths(filePaths, folder.Type) + primaryPaths, extraCandidates := partitionExtraPaths(filePaths, folder.Type, folder.Paths) rootOverrides, err := s.loadRootOverrides(ctx, folder.ID, reconcileRoots) if err != nil { return nil, fmt.Errorf("loading root overrides: %w", err) @@ -1328,7 +1335,7 @@ func (s *Scanner) scanScope( }) if ctx.Err() == nil { - extraStats := s.processExtraFiles(ctx, folder, walkRoots, extraCandidates, existingByPath) + extraStats := s.processExtraFiles(ctx, folder, extraCandidates, existingByPath) result.New += extraStats.New result.Updated += extraStats.Updated result.Unchanged += extraStats.Unchanged @@ -1658,8 +1665,8 @@ func (s *Scanner) ScanFile(ctx context.Context, filePath string, folder *models. // A local extra (Trailers/ dir, -trailer suffix, ...) bypasses identity // inference and matching entirely. - if candidate, isExtra := classifyExtraPath(cleanFile, folder.Type); isExtra { - stats := s.processExtraFiles(ctx, folder, folder.Paths, []extraCandidate{candidate}, existingByPath) + if candidate, isExtra := newWatchExtrasClassifier(folder.Type, folder.Paths).classify(cleanFile); isExtra { + stats := s.processExtraFiles(ctx, folder, []extraCandidate{candidate}, existingByPath) if stats.Errors > 0 { return fmt.Errorf("processing extra file %s failed", cleanFile) } @@ -1893,6 +1900,37 @@ func (s *Scanner) processFile( if len(updateReasons) == 0 { return actionUnchanged, nil, nil } + + // Metadata-only fast path: when the only thing that changed is the + // derived identity/grouping (root or content-group-key reassignment), + // the media bytes are untouched and existing probe data is still valid. + // Rewrite just the identity columns in place — no ffprobe, no OSHash, + // no probe-column churn. This decouples identity/grouping-scheme changes + // from probing: a library-wide group-key scheme bump (see #319) converges + // the stored keys on the next scan without a full-library ffprobe storm. + if identityOnlyFastPathEligible(existing, updateReasons) { + mf := models.MediaFile{ + MediaFolderID: folder.ID, + FilePath: filePath, + } + populateScanIdentity(&mf, filePath, folder.Type, assignment, groupAssignment, existing) + switch id, updErr := s.fileRepo.UpdateIdentity(ctx, mf); { + case updErr == nil: + mf.ID = id + if err := s.enqueueMetadataWork(ctx, folder, &mf); err != nil { + return 0, nil, fmt.Errorf("enqueueing metadata work for file %s: %w", filePath, err) + } + return actionUpdated, updateReasons, nil + case errors.Is(updErr, ErrFileNotFound): + // The row vanished between the scan-state snapshot and this + // write (concurrent delete). Fall through to the full path, + // whose upsert re-ingests the file in this scan — the old + // behavior before the fast path existed. + default: + return 0, nil, fmt.Errorf("updating identity for file %s: %w", filePath, updErr) + } + } + action := actionUpdated // Gather hints (OSHash only). hints := s.gatherHints(filePath) @@ -1919,49 +1957,7 @@ func (s *Scanner) processFile( FileModifiedAt: &fileModifiedAt, FileHash: fileHash, } - if assignment.RootPath != "" { - mf.CanonicalRootPath = filepath.Clean(assignment.RootPath) - } else if root, ok := naming.DetectCanonicalRoot(filePath, folder.Type); ok { - mf.CanonicalRootPath = filepath.Clean(root.RootPath) - } - mf.ObservedRootPath = filepath.Clean(groupAssignment.ObservedRootPath) - mf.ContentGroupKey = groupAssignment.ContentGroupKey - mf.GroupKeyVersion = groupAssignment.GroupKeyVersion - mf.BaseTitle = groupAssignment.BaseTitle - mf.BaseYear = groupAssignment.BaseYear - mf.BaseType = groupAssignment.BaseType - mf.IdentityConfidence = groupAssignment.Confidence - mf.IdentityJSON = append([]byte(nil), groupAssignment.EvidenceJSON...) - if filenameHints := naming.ParseFilename(filePath, folder.Type); filenameHints != nil && - filenameHints.Type == "series" && filenameHints.EpisodeNum > 0 { - mf.SeasonNumber = filenameHints.SeasonNum - mf.EpisodeNumber = filenameHints.EpisodeNum - } - variantHints := naming.ParseVariantHints(filePath, folder.Type) - if existing != nil && existing.EditionSource == "import" && existing.EditionKey != "" { - variantHints = &naming.VariantHints{ - EditionRaw: existing.EditionRaw, - EditionKey: existing.EditionKey, - EditionSource: existing.EditionSource, - EditionConfidence: existing.EditionConfidence, - PresentationKind: existing.PresentationKind, - PresentationGroupKey: existing.PresentationGroupKey, - PresentationPartIndex: existing.PresentationPartIndex, - MultiEpisodeStart: existing.MultiEpisodeStart, - MultiEpisodeEnd: existing.MultiEpisodeEnd, - } - } - if variantHints != nil { - mf.EditionRaw = variantHints.EditionRaw - mf.EditionKey = variantHints.EditionKey - mf.EditionConfidence = variantHints.EditionConfidence - mf.EditionSource = variantHints.EditionSource - mf.PresentationKind = variantHints.PresentationKind - mf.PresentationGroupKey = variantHints.PresentationGroupKey - mf.PresentationPartIndex = variantHints.PresentationPartIndex - mf.MultiEpisodeStart = variantHints.MultiEpisodeStart - mf.MultiEpisodeEnd = variantHints.MultiEpisodeEnd - } + populateScanIdentity(&mf, filePath, folder.Type, assignment, groupAssignment, existing) // Apply probe data if available. if probe != nil { @@ -2040,49 +2036,9 @@ func (s *Scanner) processFile( FileModifiedAt: &fileModifiedAt, FileHash: fileHash, } - if assignment.RootPath != "" { - mf.CanonicalRootPath = filepath.Clean(assignment.RootPath) - } else if root, ok := naming.DetectCanonicalRoot(filePath, folder.Type); ok { - mf.CanonicalRootPath = filepath.Clean(root.RootPath) - } - mf.ObservedRootPath = filepath.Clean(groupAssignment.ObservedRootPath) - mf.ContentGroupKey = groupAssignment.ContentGroupKey - mf.GroupKeyVersion = groupAssignment.GroupKeyVersion - mf.BaseTitle = groupAssignment.BaseTitle - mf.BaseYear = groupAssignment.BaseYear - mf.BaseType = groupAssignment.BaseType - mf.IdentityConfidence = groupAssignment.Confidence - mf.IdentityJSON = append([]byte(nil), groupAssignment.EvidenceJSON...) - if filenameHints := naming.ParseFilename(filePath, folder.Type); filenameHints != nil && - filenameHints.Type == "series" && filenameHints.EpisodeNum > 0 { - mf.SeasonNumber = filenameHints.SeasonNum - mf.EpisodeNumber = filenameHints.EpisodeNum - } - variantHints := naming.ParseVariantHints(filePath, folder.Type) - if existing, ok := existingByPath[filePath]; ok && existing != nil && existing.EditionSource == "import" && existing.EditionKey != "" { - variantHints = &naming.VariantHints{ - EditionRaw: existing.EditionRaw, - EditionKey: existing.EditionKey, - EditionSource: existing.EditionSource, - EditionConfidence: existing.EditionConfidence, - PresentationKind: existing.PresentationKind, - PresentationGroupKey: existing.PresentationGroupKey, - PresentationPartIndex: existing.PresentationPartIndex, - MultiEpisodeStart: existing.MultiEpisodeStart, - MultiEpisodeEnd: existing.MultiEpisodeEnd, - } - } - if variantHints != nil { - mf.EditionRaw = variantHints.EditionRaw - mf.EditionKey = variantHints.EditionKey - mf.EditionConfidence = variantHints.EditionConfidence - mf.EditionSource = variantHints.EditionSource - mf.PresentationKind = variantHints.PresentationKind - mf.PresentationGroupKey = variantHints.PresentationGroupKey - mf.PresentationPartIndex = variantHints.PresentationPartIndex - mf.MultiEpisodeStart = variantHints.MultiEpisodeStart - mf.MultiEpisodeEnd = variantHints.MultiEpisodeEnd - } + // This branch only runs when the path is absent from existingByPath, so + // there is no prior row to preserve import editions from. + populateScanIdentity(&mf, filePath, folder.Type, assignment, groupAssignment, nil) // Apply probe data if available. if probe != nil { @@ -2137,6 +2093,100 @@ func (s *Scanner) processFile( return action, nil, nil } +// populateScanIdentity fills mf's derived root/group/identity and +// edition/presentation columns from freshly inferred scan assignments. Every +// field it sets is derived from the file's path and sibling layout — never from +// ffprobe — so the full update path and the metadata-only update path share it. +func populateScanIdentity( + mf *models.MediaFile, + filePath string, + folderType string, + assignment fileRootAssignment, + groupAssignment fileGroupAssignment, + existing *scanStateFile, +) { + if assignment.RootPath != "" { + mf.CanonicalRootPath = filepath.Clean(assignment.RootPath) + } else if root, ok := naming.DetectCanonicalRoot(filePath, folderType); ok { + mf.CanonicalRootPath = filepath.Clean(root.RootPath) + } + mf.ObservedRootPath = filepath.Clean(groupAssignment.ObservedRootPath) + mf.ContentGroupKey = groupAssignment.ContentGroupKey + mf.GroupKeyVersion = groupAssignment.GroupKeyVersion + mf.BaseTitle = groupAssignment.BaseTitle + mf.BaseYear = groupAssignment.BaseYear + mf.BaseType = groupAssignment.BaseType + mf.IdentityConfidence = groupAssignment.Confidence + mf.IdentityJSON = append([]byte(nil), groupAssignment.EvidenceJSON...) + if filenameHints := naming.ParseFilename(filePath, folderType); filenameHints != nil && + filenameHints.Type == "series" && filenameHints.EpisodeNum > 0 { + mf.SeasonNumber = filenameHints.SeasonNum + mf.EpisodeNumber = filenameHints.EpisodeNum + } + variantHints := naming.ParseVariantHints(filePath, folderType) + if existing != nil && existing.EditionSource == "import" && existing.EditionKey != "" { + variantHints = &naming.VariantHints{ + EditionRaw: existing.EditionRaw, + EditionKey: existing.EditionKey, + EditionSource: existing.EditionSource, + EditionConfidence: existing.EditionConfidence, + PresentationKind: existing.PresentationKind, + PresentationGroupKey: existing.PresentationGroupKey, + PresentationPartIndex: existing.PresentationPartIndex, + MultiEpisodeStart: existing.MultiEpisodeStart, + MultiEpisodeEnd: existing.MultiEpisodeEnd, + } + } + if variantHints != nil { + mf.EditionRaw = variantHints.EditionRaw + mf.EditionKey = variantHints.EditionKey + mf.EditionConfidence = variantHints.EditionConfidence + mf.EditionSource = variantHints.EditionSource + mf.PresentationKind = variantHints.PresentationKind + mf.PresentationGroupKey = variantHints.PresentationGroupKey + mf.PresentationPartIndex = variantHints.PresentationPartIndex + mf.MultiEpisodeStart = variantHints.MultiEpisodeStart + mf.MultiEpisodeEnd = variantHints.MultiEpisodeEnd + } +} + +// identityOnlyUpdateReasons reports whether every update reason is a pure +// identity/grouping reclassification (root or content-group-key reassignment) +// that can be persisted without re-probing the media bytes. Any other reason — +// size/mtime change, a reappeared file, missing-probe repair, or a subtitle +// sidecar change — needs the full update path that re-reads the file. Returns +// false for an empty slice (nothing to update). +func identityOnlyUpdateReasons(reasons []string) bool { + if len(reasons) == 0 { + return false + } + for _, reason := range reasons { + switch reason { + case "group_assignment_changed", "root_assignment_changed": + default: + return false + } + } + return true +} + +// identityOnlyFastPathEligible reports whether an existing row may take the +// metadata-only update path (UpdateIdentity, no probe) for the given reasons. +// Beyond the reasons being pure identity/grouping reassignments, the row +// itself must not need the full path's side effects: +// +// - A row still linked as an extra is being reclassified as primary content +// (extras never reach processFile); only the full upsert clears the extra +// linkage so the file can re-enter matching. +// - A row without an OSHash needs the full path once — it backfills the hash +// and fetches the hash-keyed S3 intro/credits markers, which no later scan +// reason would ever repair. +func identityOnlyFastPathEligible(existing *scanStateFile, reasons []string) bool { + return identityOnlyUpdateReasons(reasons) && + existing.ExtraID == "" && + existing.FileHash != "" +} + func scanStateUpdateReasons( existing *scanStateFile, fileSize int64, diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go index 2acb7f29..29e431ed 100644 --- a/internal/scanner/scanner_test.go +++ b/internal/scanner/scanner_test.go @@ -152,6 +152,58 @@ func TestScanStateUpdateReasons_DetectsExternalSubtitleInventoryChange(t *testin } } +func TestIdentityOnlyUpdateReasons(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + reasons []string + want bool + }{ + {"empty", nil, false}, + {"group only", []string{"group_assignment_changed"}, true}, + {"root only", []string{"root_assignment_changed"}, true}, + {"group and root", []string{"group_assignment_changed", "root_assignment_changed"}, true}, + {"group plus mtime needs reprobe", []string{"group_assignment_changed", "mtime_changed"}, false}, + {"probe repair needs reprobe", []string{"probe_repair"}, false}, + {"size change needs reprobe", []string{"size_changed"}, false}, + {"was missing needs reprobe", []string{"was_missing"}, false}, + {"subtitle change is not identity-only", []string{"external_subtitle_changed"}, false}, + {"group plus subtitle needs full path", []string{"group_assignment_changed", "external_subtitle_changed"}, false}, + } + for _, tc := range cases { + if got := identityOnlyUpdateReasons(tc.reasons); got != tc.want { + t.Errorf("identityOnlyUpdateReasons(%#v) = %v, want %v", tc.reasons, got, tc.want) + } + } +} + +func TestIdentityOnlyFastPathEligible(t *testing.T) { + t.Parallel() + + identityReasons := []string{"group_assignment_changed"} + cases := []struct { + name string + existing scanStateFile + reasons []string + want bool + }{ + {"probed primary row", scanStateFile{FileHash: "abc"}, identityReasons, true}, + {"non-identity reasons need full path", scanStateFile{FileHash: "abc"}, []string{"size_changed"}, false}, + // A row still linked as an extra is being reclassified as primary; + // only the full upsert clears extra_id so matching can pick it up. + {"former extra needs full path", scanStateFile{ExtraID: "extra-1", FileHash: "abc"}, identityReasons, false}, + // A hash-less row needs the full path once to backfill OSHash and the + // hash-keyed S3 markers. + {"missing hash needs full path", scanStateFile{}, identityReasons, false}, + } + for _, tc := range cases { + if got := identityOnlyFastPathEligible(&tc.existing, tc.reasons); got != tc.want { + t.Errorf("%s: identityOnlyFastPathEligible = %v, want %v", tc.name, got, tc.want) + } + } +} + func testStringSliceContains(values []string, target string) bool { for _, value := range values { if value == target { diff --git a/internal/scanner/update_identity_db_test.go b/internal/scanner/update_identity_db_test.go new file mode 100644 index 00000000..62e7478e --- /dev/null +++ b/internal/scanner/update_identity_db_test.go @@ -0,0 +1,160 @@ +package scanner + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// TestUpdateIdentityPreservesProbeData covers the scanner's metadata-only update +// path (issue #319 hardening): rewriting a file's derived identity/grouping must +// persist the new root/group columns while leaving probe data, file bytes, and +// content linkage untouched — no ffprobe, no probe-column churn. Like every +// scan write it must clear match suppression, and it must follow folder moves. +func TestUpdateIdentityPreservesProbeData(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("ui-content-%d", suffix) + path := fmt.Sprintf("/tmp/ui-%d/Movie (2020) {tvdb-1}/Movie (2020).mkv", suffix) + probedAt := time.Now().Add(-72 * time.Hour).UTC().Truncate(time.Second) + + var folderID, movedFolderID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_folders (type, name, enabled) VALUES ('movies', 'UI Test', true) RETURNING id + `).Scan(&folderID); err != nil { + t.Fatalf("seed folder: %v", err) + } + if err := pool.QueryRow(ctx, ` + INSERT INTO media_folders (type, name, enabled) VALUES ('movies', 'UI Test Moved', true) RETURNING id + `).Scan(&movedFolderID); err != nil { + t.Fatalf("seed moved folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_files WHERE media_folder_id = ANY($1)`, []int{folderID, movedFolderID}) + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = ANY($1)`, []int{folderID, movedFolderID}) + }) + + var fileID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files ( + content_id, media_folder_id, file_path, file_size, + observed_root_path, canonical_root_path, content_group_key, group_key_version, + base_title, base_year, base_type, + codec_video, codec_audio, resolution, container, duration, bitrate, + video_tracks, audio_tracks, chapters, probe_source, probe_updated_at, + match_suppressed_at + ) VALUES ( + $1, $2, $3, 123456, + '/old/root', '/old/root', 'v1|movie|movie|2020', 1, + 'Movie', 2020, 'movie', + 'h264', 'aac', '1080p', 'mkv', 7200, 5000, + '[{"index":0}]'::jsonb, '[{"index":1}]'::jsonb, '[]'::jsonb, 'local', $4, + NOW() + ) RETURNING id + `, contentID, folderID, path, probedAt).Scan(&fileID); err != nil { + t.Fatalf("seed media file: %v", err) + } + + repo := NewFileRepository(pool) + updatedID, err := repo.UpdateIdentity(ctx, models.MediaFile{ + MediaFolderID: movedFolderID, + FilePath: path, + ObservedRootPath: "/new/root", + CanonicalRootPath: "/new/root", + ContentGroupKey: "v1|movie|anchor|tvdb-1", + GroupKeyVersion: 1, + BaseTitle: "Movie", + BaseYear: 2020, + BaseType: "movie", + }) + if err != nil { + t.Fatalf("UpdateIdentity: %v", err) + } + if updatedID != fileID { + t.Errorf("UpdateIdentity id = %d, want %d", updatedID, fileID) + } + + updated, err := repo.GetByPath(ctx, path) + if err != nil { + t.Fatalf("GetByPath after UpdateIdentity: %v", err) + } + + // Identity/grouping columns rewritten. + if updated.ContentGroupKey != "v1|movie|anchor|tvdb-1" { + t.Errorf("content_group_key = %q, want anchored form", updated.ContentGroupKey) + } + if updated.ObservedRootPath != "/new/root" { + t.Errorf("observed_root_path = %q, want /new/root", updated.ObservedRootPath) + } + if updated.CanonicalRootPath != "/new/root" { + t.Errorf("canonical_root_path = %q, want /new/root", updated.CanonicalRootPath) + } + if updated.BaseTitle != "Movie" || updated.BaseYear != 2020 || updated.BaseType != "movie" { + t.Errorf("base title/year/type = %q/%d/%q, want Movie/2020/movie", + updated.BaseTitle, updated.BaseYear, updated.BaseType) + } + if updated.MediaFolderID != movedFolderID { + t.Errorf("media_folder_id = %d, want moved folder %d", updated.MediaFolderID, movedFolderID) + } + + // Probe data and linkage preserved. + if updated.ContentID != contentID { + t.Errorf("content_id = %q, want preserved %q", updated.ContentID, contentID) + } + if updated.CodecVideo != "h264" || updated.CodecAudio != "aac" || updated.Resolution != "1080p" { + t.Errorf("probe codecs mutated: video=%q audio=%q res=%q", updated.CodecVideo, updated.CodecAudio, updated.Resolution) + } + if updated.Duration != 7200 { + t.Errorf("duration = %d, want preserved 7200", updated.Duration) + } + if updated.ProbeSource != "local" { + t.Errorf("probe_source = %q, want preserved local", updated.ProbeSource) + } + if updated.ProbeUpdatedAt == nil || !updated.ProbeUpdatedAt.Equal(probedAt) { + t.Errorf("probe_updated_at = %v, want preserved %v", updated.ProbeUpdatedAt, probedAt) + } + if len(updated.VideoTracks) != 1 || len(updated.AudioTracks) != 1 { + t.Errorf("track arrays mutated: video=%d audio=%d", len(updated.VideoTracks), len(updated.AudioTracks)) + } + if updated.FileSize != 123456 { + t.Errorf("file_size = %d, want preserved 123456", updated.FileSize) + } + + // Match suppression cleared like any other scan write, so the fresh + // identity re-enters the match backlog. + var suppressed bool + if err := pool.QueryRow(ctx, ` + SELECT match_suppressed_at IS NOT NULL FROM media_files WHERE id = $1 + `, fileID).Scan(&suppressed); err != nil { + t.Fatalf("read match_suppressed_at: %v", err) + } + if suppressed { + t.Error("match_suppressed_at still set, want cleared by identity update") + } + + // A vanished row surfaces as ErrFileNotFound so the scanner can fall back + // to the full upsert path. + if _, err := repo.UpdateIdentity(ctx, models.MediaFile{ + MediaFolderID: folderID, + FilePath: path + ".does-not-exist", + }); !errors.Is(err, ErrFileNotFound) { + t.Errorf("UpdateIdentity on missing row: err = %v, want ErrFileNotFound", err) + } +}