2537 lines
77 KiB
Go
2537 lines
77 KiB
Go
package scanner
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/Silo-Server/silo-server/internal/catalog"
|
|
"github.com/Silo-Server/silo-server/internal/models"
|
|
"github.com/Silo-Server/silo-server/internal/naming"
|
|
"github.com/Silo-Server/silo-server/internal/s3client"
|
|
)
|
|
|
|
// videoExtensions is the set of file extensions recognized as media files.
|
|
var videoExtensions = map[string]bool{
|
|
".mkv": true,
|
|
".mp4": true,
|
|
".avi": true,
|
|
".m4v": true,
|
|
".ts": true,
|
|
".wmv": true,
|
|
}
|
|
|
|
// SupportsVideoFile reports whether the given path uses a recognized media extension.
|
|
func SupportsVideoFile(filePath string) bool {
|
|
return videoExtensions[strings.ToLower(filepath.Ext(filePath))]
|
|
}
|
|
|
|
// ignoredDirNames is the set of directory names skipped during scanning.
|
|
var ignoredDirNames = map[string]bool{
|
|
".recyclebin": true,
|
|
"@recycle": true,
|
|
"@eadir": true,
|
|
".trash": true,
|
|
"#recycle": true,
|
|
"$recycle.bin": true,
|
|
".deleted": true,
|
|
".inbound": true,
|
|
".downloads": true,
|
|
}
|
|
|
|
var ignoredMovieSupplementalDirNames = map[string]bool{
|
|
"sample": true,
|
|
"samples": true,
|
|
"extra": true,
|
|
"extras": true,
|
|
"featurette": true,
|
|
"featurettes": true,
|
|
"behind the scenes": true,
|
|
"deleted scenes": true,
|
|
"trailer": true,
|
|
"trailers": true,
|
|
"subs": true,
|
|
"subtitles": true,
|
|
}
|
|
|
|
func normalizeScannerDirLabel(name string) string {
|
|
surface := strings.ToLower(strings.TrimSpace(name))
|
|
surface = strings.NewReplacer(".", " ", "_", " ", "-", " ").Replace(surface)
|
|
return strings.Join(strings.Fields(surface), " ")
|
|
}
|
|
|
|
func shouldSkipMovieSupplementalDir(path string) bool {
|
|
label := normalizeScannerDirLabel(filepath.Base(path))
|
|
if label == "" {
|
|
return false
|
|
}
|
|
return ignoredMovieSupplementalDirNames[label]
|
|
}
|
|
|
|
func shouldSkipMovieSupplementalFile(path string) bool {
|
|
baseNoExt := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
|
surface := normalizeScannerDirLabel(baseNoExt)
|
|
if surface == "" {
|
|
return false
|
|
}
|
|
if surface != "sample" && !strings.HasPrefix(surface, "sample ") && !strings.HasSuffix(surface, " sample") {
|
|
return false
|
|
}
|
|
|
|
parentTitle, parentYear, trusted := naming.ParseInferFolderTitleYear(filepath.Base(filepath.Dir(path)))
|
|
if !trusted {
|
|
return true
|
|
}
|
|
stem := naming.ParseInferMovieStem(baseNoExt, parentTitle, parentYear)
|
|
return stem.Title == "" || !naming.InferTitlesCoherent(parentTitle, stem.Title)
|
|
}
|
|
|
|
// Scanner discovers and indexes media files in media folders.
|
|
type Scanner struct {
|
|
fileRepo *FileRepository
|
|
rootSnapshotRepo *ScannedRootRepository
|
|
groupSnapshotRepo *ScannedGroupRepository
|
|
rootOverrideRepo *MediaRootOverrideRepository
|
|
groupOverrideRepo *MediaGroupOverrideRepository
|
|
locationRepo *ObservedLocationRepository
|
|
groupLocationRepo *GroupLocationRepository
|
|
folderRepo *catalog.FolderRepository
|
|
libraryRepo *catalog.LibraryItemRepository
|
|
episodeLibraryRepo *catalog.EpisodeLibraryRepository
|
|
ffprobePath string
|
|
s3Client *s3client.Client // public assets bucket (may be nil)
|
|
workers int
|
|
emptyTrashAfterScan bool
|
|
markerFetcher func(context.Context, string) *IntroCreditsMarkers
|
|
metadataQueue MetadataQueueProducer
|
|
movieQueueSyncer MovieQueueSyncer
|
|
seriesQueueSyncer SeriesQueueSyncer
|
|
}
|
|
|
|
const scanProgressLogInterval = 10 * time.Second
|
|
|
|
// MetadataQueueProducer enqueues durable initial metadata work rows after a
|
|
// successful scan upsert.
|
|
type MetadataQueueProducer interface {
|
|
EnqueueMovieFile(ctx context.Context, fileID int) error
|
|
EnqueueSeriesRoot(ctx context.Context, folderID int, observedRootPath string) error
|
|
}
|
|
|
|
// MovieQueueSyncer synchronizes pending movie-file match queue state from the
|
|
// scanner's persisted file rows.
|
|
type MovieQueueSyncer interface {
|
|
SyncForFolder(ctx context.Context, folderID int) error
|
|
SyncInScope(ctx context.Context, folderID int, scopePath string) error
|
|
}
|
|
|
|
// SeriesQueueSyncer synchronizes pending series-root match queue state from
|
|
// the scanner's persisted snapshot tables.
|
|
type SeriesQueueSyncer interface {
|
|
SyncForFolder(ctx context.Context, folderID int) error
|
|
SyncInScope(ctx context.Context, folderID int, scopePath string) error
|
|
}
|
|
|
|
// NewScanner creates a new Scanner with the given dependencies.
|
|
func NewScanner(fileRepo *FileRepository, ffprobePath string, s3Client *s3client.Client, workers int, emptyTrashAfterScan bool) *Scanner {
|
|
if workers < 1 {
|
|
workers = 8
|
|
}
|
|
return &Scanner{
|
|
fileRepo: fileRepo,
|
|
rootSnapshotRepo: NewScannedRootRepository(fileRepo.Pool()),
|
|
groupSnapshotRepo: NewScannedGroupRepository(fileRepo.Pool()),
|
|
rootOverrideRepo: NewMediaRootOverrideRepository(fileRepo.Pool()),
|
|
groupOverrideRepo: NewMediaGroupOverrideRepository(fileRepo.Pool()),
|
|
locationRepo: NewObservedLocationRepository(fileRepo.Pool()),
|
|
groupLocationRepo: NewGroupLocationRepository(fileRepo.Pool()),
|
|
folderRepo: catalog.NewFolderRepository(fileRepo.Pool()),
|
|
libraryRepo: catalog.NewLibraryItemRepository(fileRepo.Pool()),
|
|
episodeLibraryRepo: catalog.NewEpisodeLibraryRepository(fileRepo.Pool()),
|
|
ffprobePath: ffprobePath,
|
|
s3Client: s3Client,
|
|
workers: workers,
|
|
emptyTrashAfterScan: emptyTrashAfterScan,
|
|
markerFetcher: nil,
|
|
}
|
|
}
|
|
|
|
// SetSeriesQueueSyncer installs the optional pending-series root queue synchronizer.
|
|
func (s *Scanner) SetSeriesQueueSyncer(syncer SeriesQueueSyncer) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.seriesQueueSyncer = syncer
|
|
}
|
|
|
|
// SetMovieQueueSyncer installs the optional pending movie-file match queue synchronizer.
|
|
func (s *Scanner) SetMovieQueueSyncer(syncer MovieQueueSyncer) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.movieQueueSyncer = syncer
|
|
}
|
|
|
|
// SetMetadataQueueProducer installs the optional inline metadata queue producer.
|
|
func (s *Scanner) SetMetadataQueueProducer(producer MetadataQueueProducer) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.metadataQueue = producer
|
|
}
|
|
|
|
// ScanFolder walks a media folder's directory tree, discovers media files,
|
|
// probes them for technical data, and upserts them into the database.
|
|
// Files previously in the DB that no longer exist on disk are marked as missing.
|
|
func (s *Scanner) ScanFolder(ctx context.Context, folder *models.MediaFolder) (*ScanResult, error) {
|
|
watchCtx, stopWatch := s.watchFolderContext(ctx, folder.ID)
|
|
defer stopWatch()
|
|
return s.scanPaths(watchCtx, folder, folder.Paths, folder.Paths, true)
|
|
}
|
|
|
|
// ScanSubtree walks a single subtree within a media folder and reconciles only
|
|
// files that live beneath that subtree.
|
|
func (s *Scanner) ScanSubtree(ctx context.Context, folder *models.MediaFolder, subtreePath string) (*ScanResult, error) {
|
|
cleanSubtree := filepath.Clean(subtreePath)
|
|
watchCtx, stopWatch := s.watchFolderContext(ctx, folder.ID)
|
|
defer stopWatch()
|
|
return s.scanPaths(watchCtx, folder, []string{cleanSubtree}, []string{cleanSubtree}, false)
|
|
}
|
|
|
|
func isMovieLibraryType(libraryType string) bool {
|
|
switch strings.ToLower(strings.TrimSpace(libraryType)) {
|
|
case "movie", "movies":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func canonicalWalkPath(path string) (string, error) {
|
|
resolved, err := filepath.EvalSymlinks(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
abs, err := filepath.Abs(resolved)
|
|
if err == nil {
|
|
resolved = abs
|
|
}
|
|
return filepath.Clean(resolved), nil
|
|
}
|
|
|
|
func isIgnoredDirectoryPath(path string) bool {
|
|
return ignoredDirNames[strings.ToLower(filepath.Base(path))]
|
|
}
|
|
|
|
func walkLogicalTree(
|
|
ctx context.Context,
|
|
logicalPath string,
|
|
physicalPath string,
|
|
movieLibrary bool,
|
|
visitedPhysicalDirs map[string]struct{},
|
|
filePaths *[]string,
|
|
) error {
|
|
if ctx != nil {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
info, err := os.Lstat(physicalPath)
|
|
if err != nil {
|
|
slog.Warn("scanner: walk lstat failed", "path", logicalPath, "physical_path", physicalPath, "error", err)
|
|
return nil
|
|
}
|
|
|
|
if info.Mode()&os.ModeSymlink != 0 {
|
|
resolved, err := filepath.EvalSymlinks(physicalPath)
|
|
if err != nil {
|
|
slog.Warn("scanner: symlink resolve failed", "path", logicalPath, "physical_path", physicalPath, "error", err)
|
|
return nil
|
|
}
|
|
targetInfo, err := os.Stat(resolved)
|
|
if err != nil {
|
|
slog.Warn("scanner: symlink stat failed", "path", logicalPath, "resolved_path", resolved, "error", err)
|
|
return nil
|
|
}
|
|
if targetInfo.IsDir() {
|
|
return walkLogicalTree(ctx, logicalPath, resolved, movieLibrary, visitedPhysicalDirs, filePaths)
|
|
}
|
|
if movieLibrary && shouldSkipMovieSupplementalFile(logicalPath) {
|
|
return nil
|
|
}
|
|
if videoExtensions[strings.ToLower(filepath.Ext(logicalPath))] {
|
|
*filePaths = append(*filePaths, logicalPath)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if !info.IsDir() {
|
|
if movieLibrary && shouldSkipMovieSupplementalFile(logicalPath) {
|
|
return nil
|
|
}
|
|
if videoExtensions[strings.ToLower(filepath.Ext(logicalPath))] {
|
|
*filePaths = append(*filePaths, logicalPath)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
canonicalDir, err := canonicalWalkPath(physicalPath)
|
|
if err != nil {
|
|
slog.Warn("scanner: canonical path resolution failed", "path", logicalPath, "physical_path", physicalPath, "error", err)
|
|
return nil
|
|
}
|
|
if _, seen := visitedPhysicalDirs[canonicalDir]; seen {
|
|
return nil
|
|
}
|
|
visitedPhysicalDirs[canonicalDir] = struct{}{}
|
|
|
|
if isIgnoredDirectoryPath(logicalPath) {
|
|
return nil
|
|
}
|
|
if movieLibrary && shouldSkipMovieSupplementalDir(logicalPath) {
|
|
return nil
|
|
}
|
|
|
|
entries, err := os.ReadDir(physicalPath)
|
|
if err != nil {
|
|
slog.Warn("scanner: directory read failed", "path", logicalPath, "physical_path", physicalPath, "error", err)
|
|
return nil
|
|
}
|
|
for _, entry := range entries {
|
|
if ctx != nil {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
logicalChild := filepath.Join(logicalPath, entry.Name())
|
|
physicalChild := filepath.Join(physicalPath, entry.Name())
|
|
|
|
if entry.Type()&os.ModeSymlink != 0 {
|
|
resolved, err := filepath.EvalSymlinks(physicalChild)
|
|
if err != nil {
|
|
slog.Warn("scanner: symlink resolve failed", "path", logicalChild, "physical_path", physicalChild, "error", err)
|
|
continue
|
|
}
|
|
targetInfo, err := os.Stat(resolved)
|
|
if err != nil {
|
|
slog.Warn("scanner: symlink stat failed", "path", logicalChild, "resolved_path", resolved, "error", err)
|
|
continue
|
|
}
|
|
if targetInfo.IsDir() {
|
|
if err := walkLogicalTree(ctx, logicalChild, resolved, movieLibrary, visitedPhysicalDirs, filePaths); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
if movieLibrary && shouldSkipMovieSupplementalFile(logicalChild) {
|
|
continue
|
|
}
|
|
if videoExtensions[strings.ToLower(filepath.Ext(entry.Name()))] {
|
|
*filePaths = append(*filePaths, logicalChild)
|
|
}
|
|
continue
|
|
}
|
|
|
|
if entry.IsDir() {
|
|
if err := walkLogicalTree(ctx, logicalChild, physicalChild, movieLibrary, visitedPhysicalDirs, filePaths); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
|
|
if movieLibrary && shouldSkipMovieSupplementalFile(logicalChild) {
|
|
continue
|
|
}
|
|
if videoExtensions[strings.ToLower(filepath.Ext(entry.Name()))] {
|
|
*filePaths = append(*filePaths, logicalChild)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func collectLogicalFilePaths(ctx context.Context, walkRoots []string, libraryType string) ([]string, error) {
|
|
filePaths := make([]string, 0)
|
|
visitedPhysicalDirs := make(map[string]struct{})
|
|
movieLibrary := isMovieLibraryType(libraryType)
|
|
|
|
for _, rootPath := range walkRoots {
|
|
if ctx != nil {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
cleanRoot := filepath.Clean(rootPath)
|
|
if cleanRoot == "" || cleanRoot == "." {
|
|
continue
|
|
}
|
|
if err := walkLogicalTree(ctx, cleanRoot, cleanRoot, movieLibrary, visitedPhysicalDirs, &filePaths); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
return filePaths, nil
|
|
}
|
|
|
|
func (s *Scanner) scanPaths(
|
|
ctx context.Context,
|
|
folder *models.MediaFolder,
|
|
walkRoots []string,
|
|
reconcileRoots []string,
|
|
allowEmptyRootGuard bool,
|
|
) (*ScanResult, error) {
|
|
if err := s.ensureFolderEnabled(ctx, folder.ID); err != nil {
|
|
return nil, err
|
|
}
|
|
if allowEmptyRootGuard {
|
|
return s.scanFolderByRoots(ctx, folder, walkRoots, reconcileRoots)
|
|
}
|
|
|
|
result := &ScanResult{}
|
|
|
|
// Get existing files in this scan scope from the DB.
|
|
var (
|
|
existingFiles []*scanStateFile
|
|
err error
|
|
)
|
|
if allowEmptyRootGuard {
|
|
existingFiles, err = s.fileRepo.GetScanStateByFolder(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("getting existing files for folder %d: %w", folder.ID, err)
|
|
}
|
|
} else {
|
|
existingFiles, err = s.fileRepo.GetScanStateByFolderAndPathPrefix(ctx, folder.ID, reconcileRoots[0])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("getting existing files for folder %d path %q: %w", folder.ID, reconcileRoots[0], err)
|
|
}
|
|
}
|
|
|
|
// Build a set of existing file paths for quick lookup.
|
|
existingByPath := make(map[string]*scanStateFile, len(existingFiles))
|
|
for _, f := range existingFiles {
|
|
existingByPath[f.FilePath] = f
|
|
}
|
|
existingContentStatuses, err := s.loadItemStatuses(ctx, collectScanStateContentIDs(existingFiles))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading item statuses for folder %d: %w", folder.ID, err)
|
|
}
|
|
|
|
// Phase 1: Collect all media file paths.
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "walking",
|
|
Message: "Discovering media files",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
})
|
|
filePaths, walkErr := collectLogicalFilePaths(ctx, walkRoots, folder.Type)
|
|
if walkErr != nil {
|
|
return nil, fmt.Errorf("walking media roots: %w", walkErr)
|
|
}
|
|
slog.Info("scanner: discovered files",
|
|
"folder_id", folder.ID,
|
|
"scope", firstScope(reconcileRoots),
|
|
"files", len(filePaths),
|
|
)
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "processing",
|
|
Message: "Processing discovered files",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
TotalFiles: len(filePaths),
|
|
FilesDiscovered: len(filePaths),
|
|
})
|
|
|
|
// Track which paths we see on disk so we can detect missing files.
|
|
seenPaths := make(map[string]bool, len(filePaths))
|
|
for _, p := range filePaths {
|
|
seenPaths[p] = true
|
|
}
|
|
rootOverrides, err := s.loadRootOverrides(ctx, folder.ID, reconcileRoots)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading root overrides: %w", err)
|
|
}
|
|
rootInference := inferRootAssignments(filePaths, folder.Type, folder.ID, rootOverrides)
|
|
groupInference := inferGroupAssignments(filePaths, folder.Type, folder.ID, rootInference.Assignments)
|
|
groupOverrides, err := s.loadGroupOverrides(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading group overrides: %w", err)
|
|
}
|
|
applyGroupOverrides(&groupInference, groupOverrides)
|
|
result.RootObservations = rootInference.Observations
|
|
s.logRootInferenceDisagreements(rootInference.Assignments)
|
|
if err := s.reconcileScannedRoots(
|
|
ctx,
|
|
folder.ID,
|
|
reconcileRoots,
|
|
rootInference.Snapshots,
|
|
); err != nil {
|
|
return nil, fmt.Errorf("reconciling scanned roots: %w", err)
|
|
}
|
|
if _, err := s.clearLegacyLinksForUnmatchableRoots(ctx, folder.ID, result.RootObservations); err != nil {
|
|
return nil, fmt.Errorf("clearing legacy links for unmatchable roots: %w", err)
|
|
}
|
|
|
|
allowEmptyCleanup := false
|
|
if allowEmptyRootGuard && len(filePaths) == 0 && len(existingFiles) > 0 {
|
|
var err error
|
|
allowEmptyCleanup, err = s.folderRepo.ConsumeEmptyCleanupAllowance(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("checking empty cleanup confirmation for folder %d: %w", folder.ID, err)
|
|
}
|
|
if !allowEmptyCleanup {
|
|
result.EmptyRootGuarded = true
|
|
if err := s.folderRepo.SetScanWarning(ctx, folder.ID,
|
|
"empty_root",
|
|
"Scan found 0 media files; cleanup was skipped until deletion is confirmed.",
|
|
time.Now().UTC(),
|
|
); err != nil {
|
|
return nil, fmt.Errorf("recording empty-root warning for folder %d: %w", folder.ID, err)
|
|
}
|
|
return result, nil
|
|
}
|
|
}
|
|
|
|
// Phase 2: Process files concurrently with a worker pool.
|
|
var wg sync.WaitGroup
|
|
pathCh := make(chan string, s.workers)
|
|
var newCount, updatedCount, unchangedCount, errorCount, processedCount atomic.Int64
|
|
|
|
for range s.workers {
|
|
wg.Go(func() {
|
|
for path := range pathCh {
|
|
if ctx.Err() != nil {
|
|
continue // drain channel
|
|
}
|
|
action, updateReasons, processErr := s.processFile(ctx, path, folder, existingByPath, existingContentStatuses, rootInference.Assignments[path], groupInference.Assignments[path])
|
|
if processErr != nil {
|
|
slog.Error("scanner: file processing failed", "path", path, "error", processErr)
|
|
errorCount.Add(1)
|
|
continue
|
|
}
|
|
switch action {
|
|
case actionNew:
|
|
newCount.Add(1)
|
|
slog.Debug("scanner: new file added", "path", path)
|
|
case actionUpdated:
|
|
updatedCount.Add(1)
|
|
slog.Debug("scanner: file updated", "path", path, "reasons", updateReasons)
|
|
case actionUnchanged:
|
|
unchangedCount.Add(1)
|
|
}
|
|
processedCount.Add(1)
|
|
}
|
|
})
|
|
}
|
|
|
|
stopProgress := make(chan struct{})
|
|
progressDone := make(chan struct{})
|
|
go func() {
|
|
ticker := time.NewTicker(scanProgressLogInterval)
|
|
defer ticker.Stop()
|
|
defer close(progressDone)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-stopProgress:
|
|
return
|
|
case <-ticker.C:
|
|
processed := int(processedCount.Load())
|
|
total := len(filePaths)
|
|
slog.Info("scanner: processing progress",
|
|
"folder_id", folder.ID,
|
|
"scope", firstScope(reconcileRoots),
|
|
"processed", processed,
|
|
"total", total,
|
|
"new", newCount.Load(),
|
|
"updated", updatedCount.Load(),
|
|
"unchanged", unchangedCount.Load(),
|
|
"errors", errorCount.Load(),
|
|
)
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "processing",
|
|
Message: "Processing discovered files",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
TotalFiles: total,
|
|
FilesDiscovered: total,
|
|
FilesProcessed: processed,
|
|
New: int(newCount.Load()),
|
|
Updated: int(updatedCount.Load()),
|
|
Unchanged: int(unchangedCount.Load()),
|
|
Errors: int(errorCount.Load()),
|
|
})
|
|
}
|
|
}
|
|
}()
|
|
|
|
for _, p := range filePaths {
|
|
pathCh <- p
|
|
}
|
|
close(pathCh)
|
|
wg.Wait()
|
|
close(stopProgress)
|
|
<-progressDone
|
|
|
|
result.New = int(newCount.Load())
|
|
result.Updated = int(updatedCount.Load())
|
|
result.Unchanged = int(unchangedCount.Load())
|
|
result.Errors = int(errorCount.Load())
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "reconciling",
|
|
Message: "Reconciling scan state",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
TotalFiles: len(filePaths),
|
|
FilesDiscovered: len(filePaths),
|
|
FilesProcessed: int(processedCount.Load()),
|
|
New: result.New,
|
|
Updated: result.Updated,
|
|
Unchanged: result.Unchanged,
|
|
Errors: result.Errors,
|
|
})
|
|
if walkErr != nil {
|
|
return nil, walkErr
|
|
}
|
|
|
|
// If the scan was cancelled, return partial results without marking
|
|
// files as missing or deleting records — that would corrupt state.
|
|
if ctx.Err() != nil {
|
|
return result, ctx.Err()
|
|
}
|
|
|
|
if err := s.syncPresentLibraryState(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("syncing present library state for folder %d: %w", folder.ID, err)
|
|
}
|
|
|
|
if err := s.reconcileScannedGroups(ctx, folder.ID, allowEmptyRootGuard, reconcileRoots, !allowEmptyRootGuard, groupInference); err != nil {
|
|
return nil, fmt.Errorf("reconciling scanned groups: %w", err)
|
|
}
|
|
|
|
// Mark files that were in the DB but not found on disk as missing.
|
|
now := time.Now().UTC()
|
|
for _, existing := range existingFiles {
|
|
if seenPaths[existing.FilePath] {
|
|
continue
|
|
}
|
|
// Only mark as missing if not already marked.
|
|
if existing.MissingSince == nil {
|
|
if err := s.fileRepo.MarkMissing(ctx, existing.ID, now); err != nil {
|
|
slog.Error("scanner: failed to mark file missing",
|
|
"path", existing.FilePath,
|
|
"error", err,
|
|
)
|
|
result.Errors++
|
|
continue
|
|
}
|
|
}
|
|
result.Missing++
|
|
}
|
|
|
|
// Empty trash: delete all files marked as missing for this folder.
|
|
// Safe because the empty-root guard (above) returns early when 0 files
|
|
// are found on disk, so we only reach here when the root is populated.
|
|
if s.emptyTrashAfterScan {
|
|
trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err)
|
|
}
|
|
if trashed > 0 {
|
|
slog.Info("scanner: emptied trash", "folder_id", folder.ID, "deleted", trashed)
|
|
}
|
|
result.FilesDeleted += trashed
|
|
}
|
|
|
|
staleFileIDs := collectStaleRemovedPathFileIDs(existingFiles, seenPaths, reconcileRoots)
|
|
if len(filePaths) == 0 && allowEmptyCleanup {
|
|
staleFileIDs = make([]int, 0, len(existingFiles))
|
|
for _, existing := range existingFiles {
|
|
staleFileIDs = append(staleFileIDs, existing.ID)
|
|
}
|
|
}
|
|
deletedFiles, err := s.fileRepo.DeleteByIDs(ctx, staleFileIDs)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("deleting stale files for folder %d: %w", folder.ID, err)
|
|
}
|
|
result.FilesDeleted += deletedFiles
|
|
|
|
removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err)
|
|
}
|
|
result.MembershipsRemoved = removedMemberships
|
|
result.ItemsDeleted = deletedItems
|
|
|
|
if s.seriesQueueSyncer != nil {
|
|
if allowEmptyRootGuard {
|
|
if err := s.seriesQueueSyncer.SyncForFolder(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("syncing series match queue for folder %d: %w", folder.ID, err)
|
|
}
|
|
slog.Info("metadata: series root queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", "folder",
|
|
)
|
|
} else if len(reconcileRoots) > 0 {
|
|
for _, scopePath := range reconcileRoots {
|
|
if err := s.seriesQueueSyncer.SyncInScope(ctx, folder.ID, scopePath); err != nil {
|
|
return nil, fmt.Errorf("syncing series match queue for scope %q: %w", scopePath, err)
|
|
}
|
|
slog.Info("metadata: series root queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", scopePath,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
if s.movieQueueSyncer != nil {
|
|
if allowEmptyRootGuard {
|
|
if err := s.movieQueueSyncer.SyncForFolder(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("syncing movie match queue for folder %d: %w", folder.ID, err)
|
|
}
|
|
slog.Info("metadata: movie file queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", "folder",
|
|
)
|
|
} else if len(reconcileRoots) > 0 {
|
|
for _, scopePath := range reconcileRoots {
|
|
if err := s.movieQueueSyncer.SyncInScope(ctx, folder.ID, scopePath); err != nil {
|
|
return nil, fmt.Errorf("syncing movie match queue for scope %q: %w", scopePath, err)
|
|
}
|
|
slog.Info("metadata: movie file queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", scopePath,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Best-effort S3 image cleanup for orphaned items.
|
|
if s.s3Client != nil && len(orphanedImageDirs) > 0 {
|
|
bucket := s.s3Client.Bucket()
|
|
for _, dir := range orphanedImageDirs {
|
|
_, _ = s.s3Client.DeletePrefix(ctx, bucket, dir)
|
|
}
|
|
}
|
|
|
|
if allowEmptyRootGuard && (len(filePaths) > 0 || allowEmptyCleanup) {
|
|
if err := s.folderRepo.ClearScanWarning(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("clearing scan warning for folder %d: %w", folder.ID, err)
|
|
}
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
type scopedScan struct {
|
|
walkRoots []string
|
|
reconcileRoots []string
|
|
existingFiles []*scanStateFile
|
|
filePaths []string
|
|
seenPaths map[string]bool
|
|
rootInference rootInferenceResult
|
|
groupInference groupInferenceResult
|
|
result *ScanResult
|
|
}
|
|
|
|
func (s *Scanner) scanFolderByRoots(
|
|
ctx context.Context,
|
|
folder *models.MediaFolder,
|
|
walkRoots []string,
|
|
reconcileRoots []string,
|
|
) (*ScanResult, error) {
|
|
result := &ScanResult{}
|
|
roots := compactScanRoots(reconcileRoots)
|
|
if len(roots) == 0 {
|
|
roots = compactScanRoots(walkRoots)
|
|
}
|
|
|
|
pendingEmptyScopes := make([]*scopedScan, 0)
|
|
totalExisting := 0
|
|
seenAnyFiles := false
|
|
allowEmptyCleanup := false
|
|
|
|
for _, root := range roots {
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "walking",
|
|
Message: "Scanning library root",
|
|
CurrentScope: root,
|
|
})
|
|
scope, err := s.scanScope(ctx, folder, []string{root}, []string{root})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
totalExisting += len(scope.existingFiles)
|
|
mergeScanResult(result, scope.result)
|
|
|
|
if ctx.Err() != nil {
|
|
return result, ctx.Err()
|
|
}
|
|
|
|
if len(scope.filePaths) > 0 {
|
|
seenAnyFiles = true
|
|
for _, pending := range pendingEmptyScopes {
|
|
beforeErrors := pending.result.Errors
|
|
if err := s.applyScopedScan(ctx, folder, pending, false); err != nil {
|
|
return nil, err
|
|
}
|
|
mergeCleanupResult(result, pending.result, beforeErrors)
|
|
}
|
|
pendingEmptyScopes = pendingEmptyScopes[:0]
|
|
|
|
beforeErrors := scope.result.Errors
|
|
if err := s.applyScopedScan(ctx, folder, scope, false); err != nil {
|
|
return nil, err
|
|
}
|
|
mergeCleanupResult(result, scope.result, beforeErrors)
|
|
continue
|
|
}
|
|
|
|
if seenAnyFiles {
|
|
beforeErrors := scope.result.Errors
|
|
if err := s.applyScopedScan(ctx, folder, scope, false); err != nil {
|
|
return nil, err
|
|
}
|
|
mergeCleanupResult(result, scope.result, beforeErrors)
|
|
continue
|
|
}
|
|
|
|
pendingEmptyScopes = append(pendingEmptyScopes, scope)
|
|
}
|
|
|
|
if !seenAnyFiles && totalExisting > 0 {
|
|
var err error
|
|
allowEmptyCleanup, err = s.folderRepo.ConsumeEmptyCleanupAllowance(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("checking empty cleanup confirmation for folder %d: %w", folder.ID, err)
|
|
}
|
|
if !allowEmptyCleanup {
|
|
result.EmptyRootGuarded = true
|
|
if err := s.folderRepo.SetScanWarning(ctx, folder.ID,
|
|
"empty_root",
|
|
"Scan found 0 media files; cleanup was skipped until deletion is confirmed.",
|
|
time.Now().UTC(),
|
|
); err != nil {
|
|
return nil, fmt.Errorf("recording empty-root warning for folder %d: %w", folder.ID, err)
|
|
}
|
|
return result, nil
|
|
}
|
|
}
|
|
|
|
for _, pending := range pendingEmptyScopes {
|
|
beforeErrors := pending.result.Errors
|
|
if err := s.applyScopedScan(ctx, folder, pending, allowEmptyCleanup); err != nil {
|
|
return nil, err
|
|
}
|
|
mergeCleanupResult(result, pending.result, beforeErrors)
|
|
}
|
|
|
|
if ctx.Err() != nil {
|
|
return result, ctx.Err()
|
|
}
|
|
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "reconciling",
|
|
Message: "Reconciling library state",
|
|
CurrentScope: "folder",
|
|
})
|
|
if err := s.syncPresentLibraryState(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("syncing present library state for folder %d: %w", folder.ID, err)
|
|
}
|
|
|
|
if s.emptyTrashAfterScan {
|
|
trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err)
|
|
}
|
|
if trashed > 0 {
|
|
slog.Info("scanner: emptied trash", "folder_id", folder.ID, "deleted", trashed)
|
|
}
|
|
result.FilesDeleted += trashed
|
|
}
|
|
|
|
staleOutsideRoots, err := s.fileRepo.ListIDsOutsideRoots(ctx, folder.ID, roots)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("listing stale files outside configured roots for folder %d: %w", folder.ID, err)
|
|
}
|
|
deletedOutsideRoots, err := s.fileRepo.DeleteByIDs(ctx, staleOutsideRoots)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("deleting stale files outside configured roots for folder %d: %w", folder.ID, err)
|
|
}
|
|
result.FilesDeleted += deletedOutsideRoots
|
|
|
|
removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err)
|
|
}
|
|
result.MembershipsRemoved = removedMemberships
|
|
result.ItemsDeleted = deletedItems
|
|
|
|
if s.seriesQueueSyncer != nil {
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "queue_sync",
|
|
Message: "Syncing series match queue",
|
|
CurrentScope: "folder",
|
|
})
|
|
if err := s.seriesQueueSyncer.SyncForFolder(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("syncing series match queue for folder %d: %w", folder.ID, err)
|
|
}
|
|
slog.Info("metadata: series root queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", "folder",
|
|
)
|
|
}
|
|
if s.movieQueueSyncer != nil {
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "queue_sync",
|
|
Message: "Syncing movie match queue",
|
|
CurrentScope: "folder",
|
|
})
|
|
if err := s.movieQueueSyncer.SyncForFolder(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("syncing movie match queue for folder %d: %w", folder.ID, err)
|
|
}
|
|
slog.Info("metadata: movie file queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", "folder",
|
|
)
|
|
}
|
|
|
|
if s.s3Client != nil && len(orphanedImageDirs) > 0 {
|
|
bucket := s.s3Client.Bucket()
|
|
for _, dir := range orphanedImageDirs {
|
|
_, _ = s.s3Client.DeletePrefix(ctx, bucket, dir)
|
|
}
|
|
}
|
|
|
|
if seenAnyFiles || allowEmptyCleanup {
|
|
if err := s.folderRepo.ClearScanWarning(ctx, folder.ID); err != nil {
|
|
return nil, fmt.Errorf("clearing scan warning for folder %d: %w", folder.ID, err)
|
|
}
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Scanner) scanScope(
|
|
ctx context.Context,
|
|
folder *models.MediaFolder,
|
|
walkRoots []string,
|
|
reconcileRoots []string,
|
|
) (*scopedScan, error) {
|
|
result := &ScanResult{}
|
|
existingFiles, err := s.fileRepo.GetScanStateByFolderAndPathPrefix(ctx, folder.ID, reconcileRoots[0])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("getting existing files for folder %d path %q: %w", folder.ID, reconcileRoots[0], err)
|
|
}
|
|
|
|
existingByPath := make(map[string]*scanStateFile, len(existingFiles))
|
|
for _, f := range existingFiles {
|
|
existingByPath[f.FilePath] = f
|
|
}
|
|
existingContentStatuses, err := s.loadItemStatuses(ctx, collectScanStateContentIDs(existingFiles))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading item statuses for folder %d path %q: %w", folder.ID, reconcileRoots[0], err)
|
|
}
|
|
|
|
filePaths, walkErr := collectLogicalFilePaths(ctx, walkRoots, folder.Type)
|
|
if walkErr != nil {
|
|
return nil, fmt.Errorf("walking media roots for %q: %w", reconcileRoots[0], walkErr)
|
|
}
|
|
slog.Info("scanner: discovered files",
|
|
"folder_id", folder.ID,
|
|
"scope", firstScope(reconcileRoots),
|
|
"files", len(filePaths),
|
|
)
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "processing",
|
|
Message: "Processing discovered files",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
TotalFiles: len(filePaths),
|
|
FilesDiscovered: len(filePaths),
|
|
})
|
|
|
|
seenPaths := make(map[string]bool, len(filePaths))
|
|
for _, p := range filePaths {
|
|
seenPaths[p] = true
|
|
}
|
|
rootOverrides, err := s.loadRootOverrides(ctx, folder.ID, reconcileRoots)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading root overrides: %w", err)
|
|
}
|
|
rootInference := inferRootAssignments(filePaths, folder.Type, folder.ID, rootOverrides)
|
|
groupInference := inferGroupAssignments(filePaths, folder.Type, folder.ID, rootInference.Assignments)
|
|
groupOverrides, err := s.loadGroupOverrides(ctx, folder.ID)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("loading group overrides: %w", err)
|
|
}
|
|
applyGroupOverrides(&groupInference, groupOverrides)
|
|
result.RootObservations = append(result.RootObservations, rootInference.Observations...)
|
|
s.logRootInferenceDisagreements(rootInference.Assignments)
|
|
if _, err := s.clearLegacyLinksForUnmatchableRoots(ctx, folder.ID, result.RootObservations); err != nil {
|
|
return nil, fmt.Errorf("clearing legacy links for unmatchable roots: %w", err)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
pathCh := make(chan string, s.workers)
|
|
var newCount, updatedCount, unchangedCount, errorCount, processedCount atomic.Int64
|
|
|
|
for range s.workers {
|
|
wg.Go(func() {
|
|
for path := range pathCh {
|
|
if ctx.Err() != nil {
|
|
continue
|
|
}
|
|
action, updateReasons, processErr := s.processFile(ctx, path, folder, existingByPath, existingContentStatuses, rootInference.Assignments[path], groupInference.Assignments[path])
|
|
if processErr != nil {
|
|
slog.Error("scanner: file processing failed", "path", path, "error", processErr)
|
|
errorCount.Add(1)
|
|
continue
|
|
}
|
|
switch action {
|
|
case actionNew:
|
|
newCount.Add(1)
|
|
slog.Debug("scanner: new file added", "path", path)
|
|
case actionUpdated:
|
|
updatedCount.Add(1)
|
|
slog.Debug("scanner: file updated", "path", path, "reasons", updateReasons)
|
|
case actionUnchanged:
|
|
unchangedCount.Add(1)
|
|
}
|
|
processedCount.Add(1)
|
|
}
|
|
})
|
|
}
|
|
|
|
stopProgress := make(chan struct{})
|
|
progressDone := make(chan struct{})
|
|
go func() {
|
|
ticker := time.NewTicker(scanProgressLogInterval)
|
|
defer ticker.Stop()
|
|
defer close(progressDone)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-stopProgress:
|
|
return
|
|
case <-ticker.C:
|
|
processed := int(processedCount.Load())
|
|
total := len(filePaths)
|
|
slog.Info("scanner: processing progress",
|
|
"folder_id", folder.ID,
|
|
"scope", firstScope(reconcileRoots),
|
|
"processed", processed,
|
|
"total", total,
|
|
"new", newCount.Load(),
|
|
"updated", updatedCount.Load(),
|
|
"unchanged", unchangedCount.Load(),
|
|
"errors", errorCount.Load(),
|
|
)
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "processing",
|
|
Message: "Processing discovered files",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
TotalFiles: total,
|
|
FilesDiscovered: total,
|
|
FilesProcessed: processed,
|
|
New: int(newCount.Load()),
|
|
Updated: int(updatedCount.Load()),
|
|
Unchanged: int(unchangedCount.Load()),
|
|
Errors: int(errorCount.Load()),
|
|
})
|
|
}
|
|
}
|
|
}()
|
|
|
|
for _, p := range filePaths {
|
|
pathCh <- p
|
|
}
|
|
close(pathCh)
|
|
wg.Wait()
|
|
close(stopProgress)
|
|
<-progressDone
|
|
|
|
result.New = int(newCount.Load())
|
|
result.Updated = int(updatedCount.Load())
|
|
result.Unchanged = int(unchangedCount.Load())
|
|
result.Errors = int(errorCount.Load())
|
|
reportProgress(ctx, ProgressUpdate{
|
|
Phase: "reconciling",
|
|
Message: "Reconciling scan state",
|
|
CurrentScope: firstScope(reconcileRoots),
|
|
TotalFiles: len(filePaths),
|
|
FilesDiscovered: len(filePaths),
|
|
FilesProcessed: int(processedCount.Load()),
|
|
New: result.New,
|
|
Updated: result.Updated,
|
|
Unchanged: result.Unchanged,
|
|
Errors: result.Errors,
|
|
})
|
|
|
|
return &scopedScan{
|
|
walkRoots: append([]string(nil), walkRoots...),
|
|
reconcileRoots: append([]string(nil), reconcileRoots...),
|
|
existingFiles: existingFiles,
|
|
filePaths: filePaths,
|
|
seenPaths: seenPaths,
|
|
rootInference: rootInference,
|
|
groupInference: groupInference,
|
|
result: result,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Scanner) applyScopedScan(
|
|
ctx context.Context,
|
|
folder *models.MediaFolder,
|
|
scope *scopedScan,
|
|
forceDeleteAll bool,
|
|
) error {
|
|
if scope == nil || scope.result == nil {
|
|
return nil
|
|
}
|
|
|
|
if err := s.reconcileScannedRoots(
|
|
ctx,
|
|
folder.ID,
|
|
scope.reconcileRoots,
|
|
scope.rootInference.Snapshots,
|
|
); err != nil {
|
|
return fmt.Errorf("reconciling scanned roots for scope %q: %w", scope.reconcileRoots[0], err)
|
|
}
|
|
if err := s.reconcileScannedGroups(ctx, folder.ID, false, scope.reconcileRoots, true, scope.groupInference); err != nil {
|
|
return fmt.Errorf("reconciling scanned groups for scope %q: %w", scope.reconcileRoots[0], err)
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
for _, existing := range scope.existingFiles {
|
|
if scope.seenPaths[existing.FilePath] {
|
|
continue
|
|
}
|
|
if existing.MissingSince == nil {
|
|
if err := s.fileRepo.MarkMissing(ctx, existing.ID, now); err != nil {
|
|
slog.Error("scanner: failed to mark file missing",
|
|
"path", existing.FilePath,
|
|
"error", err,
|
|
)
|
|
scope.result.Errors++
|
|
continue
|
|
}
|
|
}
|
|
scope.result.Missing++
|
|
}
|
|
|
|
staleFileIDs := collectStaleRemovedPathFileIDs(scope.existingFiles, scope.seenPaths, scope.reconcileRoots)
|
|
if forceDeleteAll && len(scope.filePaths) == 0 {
|
|
staleFileIDs = make([]int, 0, len(scope.existingFiles))
|
|
for _, existing := range scope.existingFiles {
|
|
staleFileIDs = append(staleFileIDs, existing.ID)
|
|
}
|
|
}
|
|
deletedFiles, err := s.fileRepo.DeleteByIDs(ctx, staleFileIDs)
|
|
if err != nil {
|
|
return fmt.Errorf("deleting stale files for scope %q: %w", scope.reconcileRoots[0], err)
|
|
}
|
|
scope.result.FilesDeleted += deletedFiles
|
|
|
|
return nil
|
|
}
|
|
|
|
func mergeScanResult(dst *ScanResult, src *ScanResult) {
|
|
if dst == nil || src == nil {
|
|
return
|
|
}
|
|
dst.New += src.New
|
|
dst.Updated += src.Updated
|
|
dst.Unchanged += src.Unchanged
|
|
dst.Errors += src.Errors
|
|
dst.RootObservations = append(dst.RootObservations, src.RootObservations...)
|
|
dst.EmptyRootGuarded = dst.EmptyRootGuarded || src.EmptyRootGuarded
|
|
}
|
|
|
|
func firstScope(scopes []string) string {
|
|
if len(scopes) == 0 {
|
|
return ""
|
|
}
|
|
return scopes[0]
|
|
}
|
|
|
|
func mergeCleanupResult(dst *ScanResult, src *ScanResult, priorErrors int) {
|
|
if dst == nil || src == nil {
|
|
return
|
|
}
|
|
dst.Missing += src.Missing
|
|
dst.FilesDeleted += src.FilesDeleted
|
|
if src.Errors > priorErrors {
|
|
dst.Errors += src.Errors - priorErrors
|
|
}
|
|
}
|
|
|
|
func compactScanRoots(paths []string) []string {
|
|
out := make([]string, 0, len(paths))
|
|
for _, rawPath := range paths {
|
|
if strings.TrimSpace(rawPath) == "" {
|
|
continue
|
|
}
|
|
path := filepath.Clean(rawPath)
|
|
if path == "" || path == "." {
|
|
continue
|
|
}
|
|
if pathWithinAnyRoot(path, out) {
|
|
continue
|
|
}
|
|
filtered := out[:0]
|
|
for _, existing := range out {
|
|
if pathWithinAnyRoot(existing, []string{path}) {
|
|
continue
|
|
}
|
|
filtered = append(filtered, existing)
|
|
}
|
|
out = append(filtered, path)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Scanner) syncPresentLibraryState(ctx context.Context, folderID int) error {
|
|
if _, err := s.fileRepo.Pool().Exec(ctx, `
|
|
UPDATE media_files mf
|
|
SET content_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE mf.media_folder_id = $1
|
|
AND mf.missing_since IS NULL
|
|
AND mf.content_id IS NOT NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM media_items mi
|
|
WHERE mi.content_id = mf.content_id
|
|
)
|
|
`, folderID); err != nil {
|
|
return fmt.Errorf("clearing dangling content links: %w", err)
|
|
}
|
|
|
|
if _, err := s.fileRepo.Pool().Exec(ctx, `
|
|
UPDATE media_files mf
|
|
SET episode_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE mf.media_folder_id = $1
|
|
AND mf.missing_since IS NULL
|
|
AND mf.episode_id IS NOT NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM episodes e
|
|
WHERE e.content_id = mf.episode_id
|
|
)
|
|
`, folderID); err != nil {
|
|
return fmt.Errorf("clearing dangling episode links: %w", err)
|
|
}
|
|
|
|
if _, err := s.fileRepo.Pool().Exec(ctx, `
|
|
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
|
SELECT DISTINCT mf.content_id, mf.media_folder_id, NOW()
|
|
FROM media_files mf
|
|
JOIN media_items mi ON mi.content_id = mf.content_id
|
|
WHERE mf.media_folder_id = $1
|
|
AND mf.missing_since IS NULL
|
|
AND mf.content_id IS NOT NULL
|
|
ON CONFLICT (content_id, media_folder_id) DO NOTHING
|
|
`, folderID); err != nil {
|
|
return fmt.Errorf("restoring folder memberships: %w", err)
|
|
}
|
|
|
|
if _, err := s.fileRepo.Pool().Exec(ctx, `
|
|
INSERT INTO episode_libraries (episode_id, media_folder_id, first_seen_at)
|
|
SELECT mf.episode_id, mf.media_folder_id, MIN(mf.created_at)
|
|
FROM media_files mf
|
|
JOIN episodes e ON e.content_id = mf.episode_id
|
|
WHERE mf.media_folder_id = $1
|
|
AND mf.missing_since IS NULL
|
|
AND mf.episode_id IS NOT NULL
|
|
GROUP BY mf.episode_id, mf.media_folder_id
|
|
ON CONFLICT (episode_id, media_folder_id) DO NOTHING
|
|
`, folderID); err != nil {
|
|
return fmt.Errorf("restoring episode folder memberships: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Scanner) reconcileLibraryMemberships(ctx context.Context, folderID int) (int, int, []string, error) {
|
|
if s.episodeLibraryRepo != nil {
|
|
if _, err := s.episodeLibraryRepo.ReconcileFolderMembership(ctx, folderID); err != nil {
|
|
return 0, 0, nil, err
|
|
}
|
|
}
|
|
return s.libraryRepo.ReconcileFolderMembership(ctx, folderID)
|
|
}
|
|
|
|
func (s *Scanner) loadItemStatuses(ctx context.Context, contentIDs []string) (map[string]string, error) {
|
|
statuses := make(map[string]string)
|
|
if s == nil || s.fileRepo == nil || len(contentIDs) == 0 {
|
|
return statuses, nil
|
|
}
|
|
|
|
rows, err := s.fileRepo.Pool().Query(ctx, `
|
|
SELECT content_id, status
|
|
FROM media_items
|
|
WHERE content_id = ANY($1)
|
|
`, contentIDs)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("querying item statuses: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
|
|
for rows.Next() {
|
|
var contentID string
|
|
var status string
|
|
if err := rows.Scan(&contentID, &status); err != nil {
|
|
return nil, fmt.Errorf("scanning item status: %w", err)
|
|
}
|
|
statuses[contentID] = status
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("iterating item statuses: %w", err)
|
|
}
|
|
|
|
return statuses, nil
|
|
}
|
|
|
|
func collectStaleRemovedPathFileIDs(existingFiles []*scanStateFile, seenPaths map[string]bool, roots []string) []int {
|
|
ids := make([]int, 0)
|
|
for _, existing := range existingFiles {
|
|
if seenPaths[existing.FilePath] {
|
|
continue
|
|
}
|
|
if pathWithinAnyRoot(existing.FilePath, roots) {
|
|
continue
|
|
}
|
|
ids = append(ids, existing.ID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func collectScanStateContentIDs(files []*scanStateFile) []string {
|
|
contentIDs := make([]string, 0, len(files))
|
|
seen := make(map[string]struct{}, len(files))
|
|
for _, file := range files {
|
|
if file == nil || strings.TrimSpace(file.ContentID) == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[file.ContentID]; ok {
|
|
continue
|
|
}
|
|
seen[file.ContentID] = struct{}{}
|
|
contentIDs = append(contentIDs, file.ContentID)
|
|
}
|
|
return contentIDs
|
|
}
|
|
|
|
func pathWithinAnyRoot(path string, roots []string) bool {
|
|
cleanPath := filepath.Clean(path)
|
|
for _, root := range roots {
|
|
cleanRoot := filepath.Clean(root)
|
|
rel, err := filepath.Rel(cleanRoot, cleanPath)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if rel == "." || rel == "" {
|
|
return true
|
|
}
|
|
if rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ScanFile scans a single file and upserts it into the database.
|
|
func (s *Scanner) ScanFile(ctx context.Context, filePath string, folder *models.MediaFolder) error {
|
|
var stopWatch context.CancelFunc
|
|
ctx, stopWatch = s.watchFolderContext(ctx, folder.ID)
|
|
defer stopWatch()
|
|
if err := s.ensureFolderEnabled(ctx, folder.ID); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Verify the file extension is recognized.
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
if !videoExtensions[ext] {
|
|
return fmt.Errorf("unrecognized video extension: %s", ext)
|
|
}
|
|
|
|
// Look up only this specific file instead of loading the entire folder.
|
|
existingByPath := make(map[string]*scanStateFile, 1)
|
|
existing, err := s.fileRepo.GetByPath(ctx, filePath)
|
|
if err == nil {
|
|
existingByPath[filePath] = scanStateFromMediaFile(existing)
|
|
}
|
|
existingContentStatuses, err := s.loadItemStatuses(ctx, collectScanStateContentIDs([]*scanStateFile{existingByPath[filePath]}))
|
|
if err != nil {
|
|
return fmt.Errorf("loading item statuses for file: %w", err)
|
|
}
|
|
observation, ok := ObserveRoot(filePath, folder.Type)
|
|
if ok {
|
|
cleared, clearErr := s.clearLegacyLinksForUnmatchableRoots(ctx, folder.ID, []RootObservation{observation})
|
|
if clearErr != nil {
|
|
return clearErr
|
|
}
|
|
if cleared > 0 {
|
|
if _, _, _, reconcileErr := s.reconcileLibraryMemberships(ctx, folder.ID); reconcileErr != nil {
|
|
return fmt.Errorf("reconciling folder membership after clearing legacy links: %w", reconcileErr)
|
|
}
|
|
}
|
|
}
|
|
|
|
rootOverrides, err := s.loadRootOverrides(ctx, folder.ID, []string{filepath.Dir(filePath)})
|
|
if err != nil {
|
|
return fmt.Errorf("loading root overrides for file: %w", err)
|
|
}
|
|
rootInference := inferRootAssignments([]string{filePath}, folder.Type, folder.ID, rootOverrides)
|
|
s.logRootInferenceDisagreements(rootInference.Assignments)
|
|
|
|
groupInference := inferGroupAssignments([]string{filePath}, folder.Type, folder.ID, rootInference.Assignments)
|
|
groupOverrides, err := s.loadGroupOverrides(ctx, folder.ID)
|
|
if err != nil {
|
|
return fmt.Errorf("loading group overrides for file: %w", err)
|
|
}
|
|
applyGroupOverrides(&groupInference, groupOverrides)
|
|
_, _, err = s.processFile(ctx, filePath, folder, existingByPath, existingContentStatuses, rootInference.Assignments[filepath.Clean(filePath)], groupInference.Assignments[filepath.Clean(filePath)])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.reconcileScannedRoots(
|
|
ctx,
|
|
folder.ID,
|
|
nil,
|
|
rootInference.Snapshots,
|
|
); err != nil {
|
|
return fmt.Errorf("reconciling scanned root for file: %w", err)
|
|
}
|
|
scopePath := filepath.Dir(filePath)
|
|
if err := s.reconcileScannedGroups(ctx, folder.ID, false, []string{scopePath}, false, groupInference); err != nil {
|
|
return fmt.Errorf("reconciling scanned groups for file: %w", err)
|
|
}
|
|
if err := s.syncPresentLibraryState(ctx, folder.ID); err != nil {
|
|
return fmt.Errorf("syncing present library state for file: %w", err)
|
|
}
|
|
if s.seriesQueueSyncer != nil {
|
|
if err := s.seriesQueueSyncer.SyncInScope(ctx, folder.ID, scopePath); err != nil {
|
|
return fmt.Errorf("syncing series match queue for file: %w", err)
|
|
}
|
|
slog.Info("metadata: series root queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", scopePath,
|
|
)
|
|
}
|
|
if s.movieQueueSyncer != nil {
|
|
if err := s.movieQueueSyncer.SyncInScope(ctx, folder.ID, filepath.Clean(filePath)); err != nil {
|
|
return fmt.Errorf("syncing movie match queue for file: %w", err)
|
|
}
|
|
slog.Info("metadata: movie file queue sync",
|
|
"folder_id", folder.ID,
|
|
"scope", filepath.Clean(filePath),
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Scanner) watchFolderContext(ctx context.Context, folderID int) (context.Context, context.CancelFunc) {
|
|
cancel := func() {}
|
|
if ctx == nil || folderID <= 0 || s == nil || s.folderRepo == nil {
|
|
return ctx, cancel
|
|
}
|
|
|
|
watchCtx, cancel := context.WithCancel(ctx)
|
|
enabled, err := s.folderEnabledState(watchCtx, folderID)
|
|
if err != nil {
|
|
slog.Warn("scanner: failed to load folder state", "folder_id", folderID, "error", err)
|
|
cancel()
|
|
return watchCtx, cancel
|
|
}
|
|
if !enabled {
|
|
cancel()
|
|
return watchCtx, cancel
|
|
}
|
|
|
|
go func() {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-watchCtx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
enabled, err := s.folderEnabledState(watchCtx, folderID)
|
|
if err != nil {
|
|
slog.Warn("scanner: failed to refresh folder state", "folder_id", folderID, "error", err)
|
|
continue
|
|
}
|
|
if !enabled {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
return watchCtx, cancel
|
|
}
|
|
|
|
func (s *Scanner) ensureFolderEnabled(ctx context.Context, folderID int) error {
|
|
if ctx != nil {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if s == nil || s.folderRepo == nil || folderID <= 0 {
|
|
return nil
|
|
}
|
|
|
|
enabled, err := s.folderEnabledState(ctx, folderID)
|
|
switch {
|
|
case errors.Is(err, catalog.ErrFolderNotFound):
|
|
return context.Canceled
|
|
case err != nil:
|
|
return fmt.Errorf("loading folder state: %w", err)
|
|
case !enabled:
|
|
return context.Canceled
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func (s *Scanner) folderEnabledState(ctx context.Context, folderID int) (bool, error) {
|
|
if s == nil || s.folderRepo == nil || folderID <= 0 {
|
|
return true, nil
|
|
}
|
|
folder, err := s.folderRepo.GetByID(ctx, folderID)
|
|
switch {
|
|
case errors.Is(err, catalog.ErrFolderNotFound):
|
|
return false, err
|
|
case err != nil:
|
|
return false, err
|
|
case folder == nil:
|
|
return false, catalog.ErrFolderNotFound
|
|
default:
|
|
return folder.Enabled, nil
|
|
}
|
|
}
|
|
|
|
// fileAction represents what happened when processing a file.
|
|
type fileAction int
|
|
|
|
const (
|
|
actionNew fileAction = iota
|
|
actionUpdated
|
|
actionUnchanged
|
|
)
|
|
|
|
// processFile handles a single file: checks if it changed, gathers hints,
|
|
// probes it, and upserts it.
|
|
func (s *Scanner) processFile(
|
|
ctx context.Context,
|
|
filePath string,
|
|
folder *models.MediaFolder,
|
|
existingByPath map[string]*scanStateFile,
|
|
existingContentStatuses map[string]string,
|
|
assignment fileRootAssignment,
|
|
groupAssignment fileGroupAssignment,
|
|
) (fileAction, []string, error) {
|
|
// Stat the file.
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
return 0, nil, fmt.Errorf("stat %s: %w", filePath, err)
|
|
}
|
|
|
|
fileSize := info.Size()
|
|
fileModifiedAt := normalizeFileModifiedAt(info.ModTime())
|
|
|
|
// Check if unchanged (same size) only when playback-critical probe data
|
|
// is already present or this scanner cannot repair it on this node. This
|
|
// lets rescans repair legacy rows when local ffprobe metadata is available
|
|
// without forcing endless rewrites on nodes that cannot probe.
|
|
if existing, ok := existingByPath[filePath]; ok {
|
|
updateReasons := scanStateUpdateReasons(existing, fileSize, fileModifiedAt, assignment, groupAssignment, folder.Type, s.ffprobePath != "")
|
|
if shouldSkipStableConfirmedScanState(existing, existingContentStatuses[existing.ContentID], fileSize, fileModifiedAt, updateReasons, s.ffprobePath != "") {
|
|
return actionUnchanged, nil, nil
|
|
}
|
|
if len(updateReasons) == 0 {
|
|
return actionUnchanged, nil, nil
|
|
}
|
|
action := actionUpdated
|
|
// Gather hints (OSHash only).
|
|
hints := s.gatherHints(filePath)
|
|
fileHash := hints.FileHash
|
|
|
|
// Try to get probe data.
|
|
probe, probeSource := s.probeFile(ctx, filePath)
|
|
|
|
// Detect external subtitles.
|
|
externalSubs, subErr := DetectExternalSubtitles(filePath)
|
|
if subErr != nil {
|
|
slog.Warn("scanner: subtitle detection failed", "path", filePath, "error", subErr)
|
|
}
|
|
|
|
// Check for intro/credits markers from S3.
|
|
markerFetcher := s.markerFetcher
|
|
if markerFetcher == nil {
|
|
markerFetcher = s.fetchMarkers
|
|
}
|
|
markers := markerFetcher(ctx, fileHash)
|
|
|
|
// Build the media file for upsert.
|
|
mf := models.MediaFile{
|
|
MediaFolderID: folder.ID,
|
|
FilePath: filePath,
|
|
FileSize: fileSize,
|
|
FileModifiedAt: &fileModifiedAt,
|
|
FileHash: fileHash,
|
|
}
|
|
if assignment.RootPath != "" {
|
|
mf.CanonicalRootPath = filepath.Clean(assignment.RootPath)
|
|
} else if root, ok := naming.DetectCanonicalRoot(filePath, folder.Type); ok {
|
|
mf.CanonicalRootPath = filepath.Clean(root.RootPath)
|
|
}
|
|
mf.ObservedRootPath = filepath.Clean(groupAssignment.ObservedRootPath)
|
|
mf.ContentGroupKey = groupAssignment.ContentGroupKey
|
|
mf.GroupKeyVersion = groupAssignment.GroupKeyVersion
|
|
mf.BaseTitle = groupAssignment.BaseTitle
|
|
mf.BaseYear = groupAssignment.BaseYear
|
|
mf.BaseType = groupAssignment.BaseType
|
|
mf.IdentityConfidence = groupAssignment.Confidence
|
|
mf.IdentityJSON = append([]byte(nil), groupAssignment.EvidenceJSON...)
|
|
if filenameHints := naming.ParseFilename(filePath, folder.Type); filenameHints != nil &&
|
|
filenameHints.Type == "series" && filenameHints.EpisodeNum > 0 {
|
|
mf.SeasonNumber = filenameHints.SeasonNum
|
|
mf.EpisodeNumber = filenameHints.EpisodeNum
|
|
}
|
|
variantHints := naming.ParseVariantHints(filePath, folder.Type)
|
|
if existing != nil && existing.EditionSource == "import" && existing.EditionKey != "" {
|
|
variantHints = &naming.VariantHints{
|
|
EditionRaw: existing.EditionRaw,
|
|
EditionKey: existing.EditionKey,
|
|
EditionSource: existing.EditionSource,
|
|
EditionConfidence: existing.EditionConfidence,
|
|
PresentationKind: existing.PresentationKind,
|
|
PresentationGroupKey: existing.PresentationGroupKey,
|
|
PresentationPartIndex: existing.PresentationPartIndex,
|
|
MultiEpisodeStart: existing.MultiEpisodeStart,
|
|
MultiEpisodeEnd: existing.MultiEpisodeEnd,
|
|
}
|
|
}
|
|
if variantHints != nil {
|
|
mf.EditionRaw = variantHints.EditionRaw
|
|
mf.EditionKey = variantHints.EditionKey
|
|
mf.EditionConfidence = variantHints.EditionConfidence
|
|
mf.EditionSource = variantHints.EditionSource
|
|
mf.PresentationKind = variantHints.PresentationKind
|
|
mf.PresentationGroupKey = variantHints.PresentationGroupKey
|
|
mf.PresentationPartIndex = variantHints.PresentationPartIndex
|
|
mf.MultiEpisodeStart = variantHints.MultiEpisodeStart
|
|
mf.MultiEpisodeEnd = variantHints.MultiEpisodeEnd
|
|
}
|
|
|
|
// Apply probe data if available.
|
|
if probe != nil {
|
|
applyProbeData(&mf, probe, probeSource)
|
|
}
|
|
|
|
if mf.SubtitleTracks == nil {
|
|
mf.SubtitleTracks = []models.SubtitleTrack{}
|
|
}
|
|
|
|
modelExternalSubs := make([]models.ExternalSubtitle, len(externalSubs))
|
|
for i, es := range externalSubs {
|
|
modelExternalSubs[i] = models.ExternalSubtitle{
|
|
Path: es.Path,
|
|
Language: es.Language,
|
|
Format: es.Format,
|
|
Title: es.Title,
|
|
Forced: es.Forced,
|
|
}
|
|
}
|
|
mf.ExternalSubtitles = modelExternalSubs
|
|
if mf.ExternalSubtitles == nil {
|
|
mf.ExternalSubtitles = []models.ExternalSubtitle{}
|
|
}
|
|
|
|
upserted, upsertErr := s.fileRepo.Upsert(ctx, mf)
|
|
if upsertErr != nil {
|
|
return 0, nil, fmt.Errorf("upserting file %s: %w", filePath, upsertErr)
|
|
}
|
|
if err := s.enqueueMetadataWork(ctx, folder, upserted); err != nil {
|
|
return 0, nil, fmt.Errorf("enqueueing metadata work for file %s: %w", filePath, err)
|
|
}
|
|
if markers != nil {
|
|
applied, markerErr := s.fileRepo.UpsertMarkers(ctx, upserted.ID, MarkerUpdate{
|
|
IntroStart: markers.IntroStart,
|
|
IntroEnd: markers.IntroEnd,
|
|
CreditsStart: markers.CreditsStart,
|
|
CreditsEnd: markers.CreditsEnd,
|
|
MarkersSource: models.MarkerSourceS3,
|
|
})
|
|
if markerErr != nil {
|
|
return 0, nil, fmt.Errorf("upserting markers for file %s: %w", filePath, markerErr)
|
|
}
|
|
if !applied {
|
|
slog.Debug("scanner: skipped lower-priority s3 markers", "path", filePath, "hash", fileHash)
|
|
}
|
|
}
|
|
|
|
return action, updateReasons, nil
|
|
}
|
|
|
|
action := actionNew
|
|
|
|
// Gather hints (OSHash only).
|
|
hints := s.gatherHints(filePath)
|
|
fileHash := hints.FileHash
|
|
|
|
// Try to get probe data.
|
|
probe, probeSource := s.probeFile(ctx, filePath)
|
|
|
|
// Detect external subtitles.
|
|
externalSubs, subErr := DetectExternalSubtitles(filePath)
|
|
if subErr != nil {
|
|
slog.Warn("scanner: subtitle detection failed", "path", filePath, "error", subErr)
|
|
}
|
|
|
|
// Check for intro/credits markers from S3.
|
|
markerFetcher := s.markerFetcher
|
|
if markerFetcher == nil {
|
|
markerFetcher = s.fetchMarkers
|
|
}
|
|
markers := markerFetcher(ctx, fileHash)
|
|
|
|
// Build the media file for upsert.
|
|
mf := models.MediaFile{
|
|
MediaFolderID: folder.ID,
|
|
FilePath: filePath,
|
|
FileSize: fileSize,
|
|
FileModifiedAt: &fileModifiedAt,
|
|
FileHash: fileHash,
|
|
}
|
|
if assignment.RootPath != "" {
|
|
mf.CanonicalRootPath = filepath.Clean(assignment.RootPath)
|
|
} else if root, ok := naming.DetectCanonicalRoot(filePath, folder.Type); ok {
|
|
mf.CanonicalRootPath = filepath.Clean(root.RootPath)
|
|
}
|
|
mf.ObservedRootPath = filepath.Clean(groupAssignment.ObservedRootPath)
|
|
mf.ContentGroupKey = groupAssignment.ContentGroupKey
|
|
mf.GroupKeyVersion = groupAssignment.GroupKeyVersion
|
|
mf.BaseTitle = groupAssignment.BaseTitle
|
|
mf.BaseYear = groupAssignment.BaseYear
|
|
mf.BaseType = groupAssignment.BaseType
|
|
mf.IdentityConfidence = groupAssignment.Confidence
|
|
mf.IdentityJSON = append([]byte(nil), groupAssignment.EvidenceJSON...)
|
|
if filenameHints := naming.ParseFilename(filePath, folder.Type); filenameHints != nil &&
|
|
filenameHints.Type == "series" && filenameHints.EpisodeNum > 0 {
|
|
mf.SeasonNumber = filenameHints.SeasonNum
|
|
mf.EpisodeNumber = filenameHints.EpisodeNum
|
|
}
|
|
variantHints := naming.ParseVariantHints(filePath, folder.Type)
|
|
if existing, ok := existingByPath[filePath]; ok && existing != nil && existing.EditionSource == "import" && existing.EditionKey != "" {
|
|
variantHints = &naming.VariantHints{
|
|
EditionRaw: existing.EditionRaw,
|
|
EditionKey: existing.EditionKey,
|
|
EditionSource: existing.EditionSource,
|
|
EditionConfidence: existing.EditionConfidence,
|
|
PresentationKind: existing.PresentationKind,
|
|
PresentationGroupKey: existing.PresentationGroupKey,
|
|
PresentationPartIndex: existing.PresentationPartIndex,
|
|
MultiEpisodeStart: existing.MultiEpisodeStart,
|
|
MultiEpisodeEnd: existing.MultiEpisodeEnd,
|
|
}
|
|
}
|
|
if variantHints != nil {
|
|
mf.EditionRaw = variantHints.EditionRaw
|
|
mf.EditionKey = variantHints.EditionKey
|
|
mf.EditionConfidence = variantHints.EditionConfidence
|
|
mf.EditionSource = variantHints.EditionSource
|
|
mf.PresentationKind = variantHints.PresentationKind
|
|
mf.PresentationGroupKey = variantHints.PresentationGroupKey
|
|
mf.PresentationPartIndex = variantHints.PresentationPartIndex
|
|
mf.MultiEpisodeStart = variantHints.MultiEpisodeStart
|
|
mf.MultiEpisodeEnd = variantHints.MultiEpisodeEnd
|
|
}
|
|
|
|
// Apply probe data if available.
|
|
if probe != nil {
|
|
applyProbeData(&mf, probe, probeSource)
|
|
}
|
|
|
|
if mf.SubtitleTracks == nil {
|
|
mf.SubtitleTracks = []models.SubtitleTrack{}
|
|
}
|
|
|
|
// Convert external subtitles to model type.
|
|
modelExternalSubs := make([]models.ExternalSubtitle, len(externalSubs))
|
|
for i, es := range externalSubs {
|
|
modelExternalSubs[i] = models.ExternalSubtitle{
|
|
Path: es.Path,
|
|
Language: es.Language,
|
|
Format: es.Format,
|
|
Title: es.Title,
|
|
Forced: es.Forced,
|
|
}
|
|
}
|
|
mf.ExternalSubtitles = modelExternalSubs
|
|
|
|
if mf.ExternalSubtitles == nil {
|
|
mf.ExternalSubtitles = []models.ExternalSubtitle{}
|
|
}
|
|
|
|
// Upsert into DB.
|
|
upserted, upsertErr := s.fileRepo.Upsert(ctx, mf)
|
|
if upsertErr != nil {
|
|
return 0, nil, fmt.Errorf("upserting file %s: %w", filePath, upsertErr)
|
|
}
|
|
if err := s.enqueueMetadataWork(ctx, folder, upserted); err != nil {
|
|
return 0, nil, fmt.Errorf("enqueueing metadata work for file %s: %w", filePath, err)
|
|
}
|
|
if markers != nil {
|
|
applied, markerErr := s.fileRepo.UpsertMarkers(ctx, upserted.ID, MarkerUpdate{
|
|
IntroStart: markers.IntroStart,
|
|
IntroEnd: markers.IntroEnd,
|
|
CreditsStart: markers.CreditsStart,
|
|
CreditsEnd: markers.CreditsEnd,
|
|
MarkersSource: models.MarkerSourceS3,
|
|
})
|
|
if markerErr != nil {
|
|
return 0, nil, fmt.Errorf("upserting markers for file %s: %w", filePath, markerErr)
|
|
}
|
|
if !applied {
|
|
slog.Debug("scanner: skipped lower-priority s3 markers", "path", filePath, "hash", fileHash)
|
|
}
|
|
}
|
|
|
|
return action, nil, nil
|
|
}
|
|
|
|
func scanStateUpdateReasons(
|
|
existing *scanStateFile,
|
|
fileSize int64,
|
|
fileModifiedAt time.Time,
|
|
assignment fileRootAssignment,
|
|
groupAssignment fileGroupAssignment,
|
|
libraryType string,
|
|
canRepairProbe bool,
|
|
) []string {
|
|
if existing == nil {
|
|
return nil
|
|
}
|
|
|
|
reasons := make([]string, 0, 7)
|
|
if existing.FileSize != fileSize {
|
|
reasons = append(reasons, "size_changed")
|
|
}
|
|
if !sameFileModifiedAt(existing.FileModifiedAt, fileModifiedAt) {
|
|
reasons = append(reasons, "mtime_changed")
|
|
}
|
|
if existing.MissingSince != nil {
|
|
reasons = append(reasons, "was_missing")
|
|
}
|
|
if canRepairProbe && needsCriticalProbeRepairScanState(existing) {
|
|
reasons = append(reasons, "probe_repair")
|
|
}
|
|
if hasMissingExternalSubtitlePath(existing.ExternalSubtitlePaths) {
|
|
reasons = append(reasons, "external_subtitle_missing")
|
|
}
|
|
if scanStateRootAssignmentChanged(existing, assignment, libraryType) {
|
|
reasons = append(reasons, "root_assignment_changed")
|
|
}
|
|
if scanStateGroupAssignmentChanged(existing, groupAssignment) {
|
|
reasons = append(reasons, "group_assignment_changed")
|
|
}
|
|
return reasons
|
|
}
|
|
|
|
func hasMissingExternalSubtitlePath(paths []string) bool {
|
|
for _, path := range paths {
|
|
if strings.TrimSpace(path) == "" {
|
|
continue
|
|
}
|
|
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func shouldSkipStableConfirmedScanState(
|
|
existing *scanStateFile,
|
|
itemStatus string,
|
|
fileSize int64,
|
|
fileModifiedAt time.Time,
|
|
updateReasons []string,
|
|
canRepairProbe bool,
|
|
) bool {
|
|
if existing == nil || existing.ContentID == "" {
|
|
return false
|
|
}
|
|
if !strings.EqualFold(strings.TrimSpace(itemStatus), "matched") {
|
|
return false
|
|
}
|
|
if existing.FileSize != fileSize {
|
|
return false
|
|
}
|
|
if !sameFileModifiedAt(existing.FileModifiedAt, fileModifiedAt) {
|
|
return false
|
|
}
|
|
if existing.MissingSince != nil {
|
|
return false
|
|
}
|
|
if canRepairProbe && needsCriticalProbeRepairScanState(existing) {
|
|
return false
|
|
}
|
|
if len(updateReasons) > 0 {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func scannerUpdateReasons(
|
|
existing *models.MediaFile,
|
|
fileSize int64,
|
|
fileModifiedAt time.Time,
|
|
assignment fileRootAssignment,
|
|
groupAssignment fileGroupAssignment,
|
|
libraryType string,
|
|
canRepairProbe bool,
|
|
) []string {
|
|
if existing == nil {
|
|
return nil
|
|
}
|
|
|
|
reasons := make([]string, 0, 6)
|
|
if existing.FileSize != fileSize {
|
|
reasons = append(reasons, "size_changed")
|
|
}
|
|
if !sameFileModifiedAt(existing.FileModifiedAt, fileModifiedAt) {
|
|
reasons = append(reasons, "mtime_changed")
|
|
}
|
|
if existing.MissingSince != nil {
|
|
reasons = append(reasons, "was_missing")
|
|
}
|
|
if canRepairProbe && NeedsCriticalProbeRepair(existing) {
|
|
reasons = append(reasons, "probe_repair")
|
|
}
|
|
if rootAssignmentChanged(existing, assignment, libraryType) {
|
|
reasons = append(reasons, "root_assignment_changed")
|
|
}
|
|
if groupAssignmentChanged(existing, groupAssignment) {
|
|
reasons = append(reasons, "group_assignment_changed")
|
|
}
|
|
return reasons
|
|
}
|
|
|
|
func shouldSkipStableConfirmedFile(
|
|
existing *models.MediaFile,
|
|
itemStatus string,
|
|
fileSize int64,
|
|
fileModifiedAt time.Time,
|
|
updateReasons []string,
|
|
canRepairProbe bool,
|
|
) bool {
|
|
if existing == nil || existing.ContentID == "" {
|
|
return false
|
|
}
|
|
if !strings.EqualFold(strings.TrimSpace(itemStatus), "matched") {
|
|
return false
|
|
}
|
|
if existing.FileSize != fileSize {
|
|
return false
|
|
}
|
|
if !sameFileModifiedAt(existing.FileModifiedAt, fileModifiedAt) {
|
|
return false
|
|
}
|
|
if existing.MissingSince != nil {
|
|
return false
|
|
}
|
|
if canRepairProbe && NeedsCriticalProbeRepair(existing) {
|
|
return false
|
|
}
|
|
if len(updateReasons) > 0 {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func sameFileModifiedAt(existing *time.Time, current time.Time) bool {
|
|
if existing == nil {
|
|
return false
|
|
}
|
|
return normalizeFileModifiedAt(*existing).Equal(normalizeFileModifiedAt(current))
|
|
}
|
|
|
|
func normalizeFileModifiedAt(ts time.Time) time.Time {
|
|
return ts.UTC().Truncate(time.Microsecond)
|
|
}
|
|
|
|
func needsCriticalProbeRepairScanState(file *scanStateFile) bool {
|
|
if file == nil {
|
|
return true
|
|
}
|
|
if strings.TrimSpace(file.ProbeSource) == "" || file.ProbeUpdatedAt == nil {
|
|
return true
|
|
}
|
|
if file.Duration <= 0 {
|
|
return true
|
|
}
|
|
if strings.TrimSpace(file.Container) == "" {
|
|
return true
|
|
}
|
|
if strings.TrimSpace(file.CodecVideo) == "" {
|
|
return true
|
|
}
|
|
if strings.TrimSpace(file.CodecAudio) == "" {
|
|
return true
|
|
}
|
|
if strings.TrimSpace(file.Resolution) == "" {
|
|
return true
|
|
}
|
|
if !file.HasVideoTracks {
|
|
return true
|
|
}
|
|
if !file.HasAudioTracks {
|
|
return true
|
|
}
|
|
if !file.HasChapters {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (s *Scanner) enqueueMetadataWork(ctx context.Context, folder *models.MediaFolder, file *models.MediaFile) error {
|
|
if s == nil || s.metadataQueue == nil || folder == nil || file == nil {
|
|
return nil
|
|
}
|
|
|
|
switch strings.ToLower(strings.TrimSpace(folder.Type)) {
|
|
case "movie", "movies":
|
|
if err := s.metadataQueue.EnqueueMovieFile(ctx, file.ID); err != nil {
|
|
return err
|
|
}
|
|
slog.Debug("metadata queue: movie file enqueued",
|
|
"folder_id", folder.ID,
|
|
"file_id", file.ID,
|
|
"path", file.FilePath,
|
|
)
|
|
case "series", "tv", "show", "tvshows":
|
|
if strings.TrimSpace(file.ObservedRootPath) == "" {
|
|
return nil
|
|
}
|
|
if err := s.metadataQueue.EnqueueSeriesRoot(ctx, folder.ID, file.ObservedRootPath); err != nil {
|
|
return err
|
|
}
|
|
slog.Debug("metadata queue: series root touched",
|
|
"folder_id", folder.ID,
|
|
"observed_root_path", file.ObservedRootPath,
|
|
"file_id", file.ID,
|
|
)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Scanner) reconcileScannedGroups(
|
|
ctx context.Context,
|
|
folderID int,
|
|
fullFolderScan bool,
|
|
scopeRoots []string,
|
|
deleteMissingInScope bool,
|
|
groups groupInferenceResult,
|
|
) error {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
if s.groupSnapshotRepo != nil {
|
|
if fullFolderScan {
|
|
if err := s.groupSnapshotRepo.ReplaceForFolder(ctx, folderID, groups.ScannedGroups); err != nil {
|
|
return err
|
|
}
|
|
} else if deleteMissingInScope {
|
|
for _, scope := range scopeRoots {
|
|
if err := s.groupSnapshotRepo.ReplaceInScope(ctx, folderID, scope, filterScannedGroupsByScope(groups.ScannedGroups, scope)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
if err := s.groupSnapshotRepo.UpsertMany(ctx, groups.ScannedGroups); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if s.locationRepo != nil {
|
|
if fullFolderScan {
|
|
if err := s.locationRepo.ReplaceForFolder(ctx, folderID, groups.Locations); err != nil {
|
|
return err
|
|
}
|
|
} else if deleteMissingInScope {
|
|
for _, scope := range scopeRoots {
|
|
if err := s.locationRepo.ReplaceInScope(ctx, folderID, scope, filterObservedLocationsByScope(groups.Locations, scope)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
if err := s.locationRepo.UpsertMany(ctx, groups.Locations); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if s.groupLocationRepo != nil {
|
|
if fullFolderScan {
|
|
if err := s.groupLocationRepo.ReplaceForFolder(ctx, folderID, groups.GroupLocations); err != nil {
|
|
return err
|
|
}
|
|
} else if deleteMissingInScope {
|
|
for _, scope := range scopeRoots {
|
|
if err := s.groupLocationRepo.ReplaceInScope(ctx, folderID, scope, filterGroupLocationsByScope(groups.GroupLocations, scope)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
if err := s.groupLocationRepo.UpsertMany(ctx, groups.GroupLocations); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func filterScannedGroupsByScope(groups []models.ScannedMediaGroup, scopePath string) []models.ScannedMediaGroup {
|
|
filtered := make([]models.ScannedMediaGroup, 0, len(groups))
|
|
for _, group := range groups {
|
|
if pathWithinAnyRoot(group.SampleObservedRootPath, []string{scopePath}) {
|
|
filtered = append(filtered, group)
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func filterObservedLocationsByScope(locations []models.ObservedMediaLocation, scopePath string) []models.ObservedMediaLocation {
|
|
filtered := make([]models.ObservedMediaLocation, 0, len(locations))
|
|
for _, location := range locations {
|
|
if pathWithinAnyRoot(location.ObservedRootPath, []string{scopePath}) {
|
|
filtered = append(filtered, location)
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func filterGroupLocationsByScope(locations []models.MediaGroupLocation, scopePath string) []models.MediaGroupLocation {
|
|
filtered := make([]models.MediaGroupLocation, 0, len(locations))
|
|
for _, location := range locations {
|
|
if pathWithinAnyRoot(location.ObservedRootPath, []string{scopePath}) {
|
|
filtered = append(filtered, location)
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func (s *Scanner) reconcileScannedRoots(
|
|
ctx context.Context,
|
|
folderID int,
|
|
scopeRoots []string,
|
|
roots []models.ScannedMediaRoot,
|
|
) error {
|
|
if s == nil || s.rootSnapshotRepo == nil {
|
|
return nil
|
|
}
|
|
|
|
seenByScope := make(map[string][]string, len(scopeRoots))
|
|
for _, scope := range scopeRoots {
|
|
seenByScope[scope] = []string{}
|
|
}
|
|
|
|
for _, root := range roots {
|
|
for scope := range seenByScope {
|
|
if pathWithinAnyRoot(root.RootPath, []string{scope}) {
|
|
seenByScope[scope] = append(seenByScope[scope], root.RootPath)
|
|
}
|
|
}
|
|
}
|
|
if err := s.rootSnapshotRepo.UpsertMany(ctx, roots); err != nil {
|
|
return err
|
|
}
|
|
|
|
for scope, seenRoots := range seenByScope {
|
|
if err := s.rootSnapshotRepo.DeleteMissingInScope(ctx, folderID, scope, seenRoots); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Scanner) loadRootOverrides(
|
|
ctx context.Context,
|
|
folderID int,
|
|
scopeRoots []string,
|
|
) (map[string]models.MediaRootOverride, error) {
|
|
overridesByRoot := map[string]models.MediaRootOverride{}
|
|
if s == nil || s.rootOverrideRepo == nil || folderID <= 0 {
|
|
return overridesByRoot, nil
|
|
}
|
|
|
|
overrides, err := s.rootOverrideRepo.ListByFolder(ctx, folderID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, override := range overrides {
|
|
if len(scopeRoots) > 0 && !pathWithinAnyRoot(override.RootPath, scopeRoots) {
|
|
continue
|
|
}
|
|
overridesByRoot[filepath.Clean(override.RootPath)] = override
|
|
}
|
|
return overridesByRoot, nil
|
|
}
|
|
|
|
func (s *Scanner) loadGroupOverrides(
|
|
ctx context.Context,
|
|
folderID int,
|
|
) (map[string]models.MediaGroupOverride, error) {
|
|
overridesByKey := map[string]models.MediaGroupOverride{}
|
|
if s == nil || s.groupOverrideRepo == nil || folderID <= 0 {
|
|
return overridesByKey, nil
|
|
}
|
|
|
|
overrides, err := s.groupOverrideRepo.ListByFolder(ctx, folderID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, override := range overrides {
|
|
overridesByKey[groupOverrideKey(override.GroupKeyVersion, override.ContentGroupKey)] = override
|
|
}
|
|
return overridesByKey, nil
|
|
}
|
|
|
|
func (s *Scanner) logRootInferenceDisagreements(assignments map[string]fileRootAssignment) {
|
|
for _, assignment := range assignments {
|
|
if assignment.LegacyRootPath == "" {
|
|
continue
|
|
}
|
|
if assignment.LegacyRootPath == assignment.RootPath && assignment.LegacyType == assignment.InferredType {
|
|
continue
|
|
}
|
|
slog.Info("scanner: root inference disagreement",
|
|
"file_path", assignment.FilePath,
|
|
"legacy_root_path", assignment.LegacyRootPath,
|
|
"inferred_root_path", assignment.RootPath,
|
|
"legacy_type", assignment.LegacyType,
|
|
"inferred_type", assignment.InferredType,
|
|
"wrapper_collapsed", assignment.WrapperCollapsed,
|
|
"promoted_ancestor", assignment.PromotedAncestor,
|
|
)
|
|
}
|
|
}
|
|
|
|
func scanStateRootAssignmentChanged(existing *scanStateFile, assignment fileRootAssignment, libraryType string) bool {
|
|
if existing == nil {
|
|
return true
|
|
}
|
|
expectedRoot := assignment.RootPath
|
|
if expectedRoot == "" {
|
|
if root, ok := naming.DetectCanonicalRoot(existing.FilePath, libraryType); ok {
|
|
expectedRoot = filepath.Clean(root.RootPath)
|
|
}
|
|
}
|
|
if filepath.Clean(existing.CanonicalRootPath) != filepath.Clean(expectedRoot) {
|
|
return true
|
|
}
|
|
|
|
hints := naming.ParseVariantHints(existing.FilePath, libraryType)
|
|
if existing.EditionSource == "import" && existing.EditionKey != "" {
|
|
hints = &naming.VariantHints{
|
|
EditionRaw: existing.EditionRaw,
|
|
EditionKey: existing.EditionKey,
|
|
EditionSource: existing.EditionSource,
|
|
EditionConfidence: existing.EditionConfidence,
|
|
PresentationKind: existing.PresentationKind,
|
|
PresentationGroupKey: existing.PresentationGroupKey,
|
|
PresentationPartIndex: existing.PresentationPartIndex,
|
|
MultiEpisodeStart: existing.MultiEpisodeStart,
|
|
MultiEpisodeEnd: existing.MultiEpisodeEnd,
|
|
}
|
|
}
|
|
if hints == nil {
|
|
hints = &naming.VariantHints{}
|
|
}
|
|
if existing.EditionRaw != hints.EditionRaw ||
|
|
existing.EditionKey != hints.EditionKey ||
|
|
existing.EditionSource != hints.EditionSource ||
|
|
existing.PresentationKind != hints.PresentationKind ||
|
|
existing.PresentationGroupKey != hints.PresentationGroupKey ||
|
|
existing.PresentationPartIndex != hints.PresentationPartIndex ||
|
|
existing.MultiEpisodeStart != hints.MultiEpisodeStart ||
|
|
existing.MultiEpisodeEnd != hints.MultiEpisodeEnd {
|
|
return true
|
|
}
|
|
switch {
|
|
case existing.EditionConfidence == nil && hints.EditionConfidence == nil:
|
|
return false
|
|
case existing.EditionConfidence == nil || hints.EditionConfidence == nil:
|
|
return true
|
|
default:
|
|
return *existing.EditionConfidence != *hints.EditionConfidence
|
|
}
|
|
}
|
|
|
|
func scanStateGroupAssignmentChanged(existing *scanStateFile, assignment fileGroupAssignment) bool {
|
|
if existing == nil {
|
|
return true
|
|
}
|
|
if filepath.Clean(existing.ObservedRootPath) != filepath.Clean(assignment.ObservedRootPath) {
|
|
return true
|
|
}
|
|
if existing.ContentGroupKey != assignment.ContentGroupKey ||
|
|
existing.GroupKeyVersion != assignment.GroupKeyVersion ||
|
|
existing.BaseTitle != assignment.BaseTitle ||
|
|
existing.BaseYear != assignment.BaseYear ||
|
|
existing.BaseType != assignment.BaseType ||
|
|
existing.IdentityConfidence != assignment.Confidence {
|
|
return true
|
|
}
|
|
return !identityEvidenceEqual(existing.IdentityJSON, assignment.EvidenceJSON)
|
|
}
|
|
|
|
func rootAssignmentChanged(existing *models.MediaFile, assignment fileRootAssignment, libraryType string) bool {
|
|
if existing == nil {
|
|
return true
|
|
}
|
|
expectedRoot := assignment.RootPath
|
|
if expectedRoot == "" {
|
|
if root, ok := naming.DetectCanonicalRoot(existing.FilePath, libraryType); ok {
|
|
expectedRoot = filepath.Clean(root.RootPath)
|
|
}
|
|
}
|
|
if filepath.Clean(existing.CanonicalRootPath) != filepath.Clean(expectedRoot) {
|
|
return true
|
|
}
|
|
|
|
hints := naming.ParseVariantHints(existing.FilePath, libraryType)
|
|
if existing.EditionSource == "import" && existing.EditionKey != "" {
|
|
hints = &naming.VariantHints{
|
|
EditionRaw: existing.EditionRaw,
|
|
EditionKey: existing.EditionKey,
|
|
EditionSource: existing.EditionSource,
|
|
EditionConfidence: existing.EditionConfidence,
|
|
PresentationKind: existing.PresentationKind,
|
|
PresentationGroupKey: existing.PresentationGroupKey,
|
|
PresentationPartIndex: existing.PresentationPartIndex,
|
|
MultiEpisodeStart: existing.MultiEpisodeStart,
|
|
MultiEpisodeEnd: existing.MultiEpisodeEnd,
|
|
}
|
|
}
|
|
if hints == nil {
|
|
hints = &naming.VariantHints{}
|
|
}
|
|
if existing.EditionRaw != hints.EditionRaw ||
|
|
existing.EditionKey != hints.EditionKey ||
|
|
existing.EditionSource != hints.EditionSource ||
|
|
existing.PresentationKind != hints.PresentationKind ||
|
|
existing.PresentationGroupKey != hints.PresentationGroupKey ||
|
|
existing.PresentationPartIndex != hints.PresentationPartIndex ||
|
|
existing.MultiEpisodeStart != hints.MultiEpisodeStart ||
|
|
existing.MultiEpisodeEnd != hints.MultiEpisodeEnd {
|
|
return true
|
|
}
|
|
switch {
|
|
case existing.EditionConfidence == nil && hints.EditionConfidence == nil:
|
|
return false
|
|
case existing.EditionConfidence == nil || hints.EditionConfidence == nil:
|
|
return true
|
|
default:
|
|
return *existing.EditionConfidence != *hints.EditionConfidence
|
|
}
|
|
}
|
|
|
|
func groupAssignmentChanged(existing *models.MediaFile, assignment fileGroupAssignment) bool {
|
|
if existing == nil {
|
|
return true
|
|
}
|
|
if filepath.Clean(existing.ObservedRootPath) != filepath.Clean(assignment.ObservedRootPath) {
|
|
return true
|
|
}
|
|
if existing.ContentGroupKey != assignment.ContentGroupKey ||
|
|
existing.GroupKeyVersion != assignment.GroupKeyVersion ||
|
|
existing.BaseTitle != assignment.BaseTitle ||
|
|
existing.BaseYear != assignment.BaseYear ||
|
|
existing.BaseType != assignment.BaseType ||
|
|
existing.IdentityConfidence != assignment.Confidence {
|
|
return true
|
|
}
|
|
return !identityEvidenceEqual(existing.IdentityJSON, assignment.EvidenceJSON)
|
|
}
|
|
|
|
func identityEvidenceEqual(existing, expected []byte) bool {
|
|
if bytes.Equal(existing, expected) {
|
|
return true
|
|
}
|
|
if len(bytes.TrimSpace(existing)) == 0 || len(bytes.TrimSpace(expected)) == 0 {
|
|
return len(bytes.TrimSpace(existing)) == 0 && len(bytes.TrimSpace(expected)) == 0
|
|
}
|
|
|
|
var existingValue any
|
|
if err := json.Unmarshal(existing, &existingValue); err != nil {
|
|
return false
|
|
}
|
|
var expectedValue any
|
|
if err := json.Unmarshal(expected, &expectedValue); err != nil {
|
|
return false
|
|
}
|
|
|
|
return reflect.DeepEqual(existingValue, expectedValue)
|
|
}
|
|
|
|
func applyProbeData(mf *models.MediaFile, probe *ProbeData, probeSource string) {
|
|
mf.CodecVideo = probe.CodecVideo
|
|
mf.CodecAudio = probe.CodecAudio
|
|
mf.Resolution = probe.Resolution
|
|
mf.AudioChannels = probe.AudioChannels
|
|
mf.HDR = probe.HDR
|
|
mf.Container = probe.Container
|
|
mf.Duration = probe.Duration
|
|
mf.Bitrate = probe.Bitrate
|
|
mf.ProbeSource = probeSource
|
|
|
|
now := time.Now().UTC()
|
|
mf.ProbeUpdatedAt = &now
|
|
|
|
videoTracks := make([]models.VideoTrack, len(probe.VideoTracks))
|
|
for i, vt := range probe.VideoTracks {
|
|
videoTracks[i] = models.VideoTrack{
|
|
Title: vt.Title,
|
|
Codec: vt.Codec,
|
|
DolbyVision: vt.DolbyVision,
|
|
Profile: vt.Profile,
|
|
Level: vt.Level,
|
|
Width: vt.Width,
|
|
Height: vt.Height,
|
|
AspectRatio: vt.AspectRatio,
|
|
Interlaced: vt.Interlaced,
|
|
FrameRate: vt.FrameRate,
|
|
Bitrate: vt.Bitrate,
|
|
VideoRange: vt.VideoRange,
|
|
ColorPrimaries: vt.ColorPrimaries,
|
|
ColorSpace: vt.ColorSpace,
|
|
ColorTransfer: vt.ColorTransfer,
|
|
BitDepth: vt.BitDepth,
|
|
PixelFormat: vt.PixelFormat,
|
|
ReferenceFrames: vt.ReferenceFrames,
|
|
}
|
|
}
|
|
mf.VideoTracks = videoTracks
|
|
|
|
audioTracks := make([]models.AudioTrack, len(probe.AudioTracks))
|
|
for i, at := range probe.AudioTracks {
|
|
audioTracks[i] = models.AudioTrack{
|
|
Title: at.Title,
|
|
EmbeddedTitle: at.EmbeddedTitle,
|
|
Language: at.Language,
|
|
Codec: at.Codec,
|
|
Layout: at.Layout,
|
|
Channels: at.Channels,
|
|
Bitrate: at.Bitrate,
|
|
SampleRate: at.SampleRate,
|
|
BitDepth: at.BitDepth,
|
|
Default: at.Default,
|
|
}
|
|
}
|
|
mf.AudioTracks = audioTracks
|
|
|
|
subtitleTracks := make([]models.SubtitleTrack, len(probe.SubtitleTracks))
|
|
for i, st := range probe.SubtitleTracks {
|
|
subtitleTracks[i] = models.SubtitleTrack{
|
|
Index: st.Index,
|
|
Language: st.Language,
|
|
Codec: st.Codec,
|
|
Title: st.Title,
|
|
EmbeddedTitle: st.EmbeddedTitle,
|
|
Resolution: st.Resolution,
|
|
Forced: st.Forced,
|
|
Default: st.Default,
|
|
HearingImpaired: st.HearingImpaired,
|
|
}
|
|
}
|
|
mf.SubtitleTracks = subtitleTracks
|
|
|
|
chapters := make([]models.MediaChapter, len(probe.Chapters))
|
|
for i, chapter := range probe.Chapters {
|
|
chapters[i] = models.MediaChapter{
|
|
Index: chapter.Index,
|
|
Title: chapter.Title,
|
|
StartSeconds: chapter.StartSeconds,
|
|
EndSeconds: chapter.EndSeconds,
|
|
Source: chapter.Source,
|
|
}
|
|
}
|
|
mf.Chapters = chapters
|
|
}
|
|
|
|
// clearLegacyLinksForUnmatchableRoots was previously used to clear content
|
|
// links for files under roots without embedded folder IDs. With the library
|
|
// matching redesign, roots without folder IDs are now valid and create
|
|
// pending items, so this function is a no-op.
|
|
// TODO: remove callers and delete this function
|
|
func (s *Scanner) clearLegacyLinksForUnmatchableRoots(_ context.Context, _ int, _ []RootObservation) (int, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
// gatherHints computes the OSHash for a media file.
|
|
func (s *Scanner) gatherHints(filePath string) FileHints {
|
|
hints := FileHints{}
|
|
hash, err := ComputeOSHash(filePath)
|
|
if err != nil {
|
|
slog.Warn("scanner: OSHash computation failed", "path", filePath, "error", err)
|
|
} else {
|
|
hints.FileHash = hash
|
|
}
|
|
return hints
|
|
}
|
|
|
|
// probeFile attempts to get probe data by running local ffprobe.
|
|
func (s *Scanner) probeFile(ctx context.Context, filePath string) (*ProbeData, string) {
|
|
if s.ffprobePath != "" {
|
|
probe, err := ProbeFile(ctx, s.ffprobePath, filePath)
|
|
if err != nil {
|
|
slog.Warn("scanner: ffprobe failed", "path", filePath, "error", err)
|
|
return nil, "local"
|
|
}
|
|
return probe, "local"
|
|
}
|
|
|
|
return nil, "local"
|
|
}
|
|
|
|
// fetchMarkers checks S3 for intro/credits markers for the given file hash.
|
|
func (s *Scanner) fetchMarkers(ctx context.Context, fileHash string) *IntroCreditsMarkers {
|
|
if fileHash == "" || s.s3Client == nil {
|
|
return nil
|
|
}
|
|
|
|
key := fmt.Sprintf("markers/%s.json", fileHash)
|
|
data, err := s.s3Client.GetObject(ctx, s.s3Client.Bucket(), key)
|
|
if err != nil {
|
|
// Not found is expected; don't log it.
|
|
return nil
|
|
}
|
|
|
|
var markers IntroCreditsMarkers
|
|
if err := json.Unmarshal(data, &markers); err != nil {
|
|
slog.Warn("scanner: markers JSON parse failed",
|
|
"hash", fileHash,
|
|
"error", err,
|
|
)
|
|
return nil
|
|
}
|
|
|
|
return &markers
|
|
}
|