diff --git a/internal/api/handlers/libraries.go b/internal/api/handlers/libraries.go index 195e41af..f9c5c2b4 100644 --- a/internal/api/handlers/libraries.go +++ b/internal/api/handlers/libraries.go @@ -1249,6 +1249,7 @@ func scanRunResultFromIngest(result *libraryingest.Result) *evt.ScanRunResult { resp.Updated = result.ScanResult.Updated resp.Unchanged = result.ScanResult.Unchanged resp.Missing = result.ScanResult.Missing + resp.MissingSkippedProtected = result.ScanResult.MissingSkippedProtected resp.FilesDeleted = result.ScanResult.FilesDeleted resp.MembershipsRemoved = result.ScanResult.MembershipsRemoved resp.ItemsDeleted = result.ScanResult.ItemsDeleted diff --git a/internal/events/scan_registry.go b/internal/events/scan_registry.go index 9eb4da35..9dff37ee 100644 --- a/internal/events/scan_registry.go +++ b/internal/events/scan_registry.go @@ -7,24 +7,29 @@ import ( ) type ScanRunResult struct { - New int `json:"new"` - Updated int `json:"updated"` - Unchanged int `json:"unchanged"` - Missing int `json:"missing"` - FilesDeleted int `json:"files_deleted"` - MembershipsRemoved int `json:"memberships_removed"` - ItemsDeleted int `json:"items_deleted"` - MatchedFiles int `json:"matched_files"` - RetriedItems int `json:"retried_items"` - StillUnmatchedWarnings int `json:"still_unmatched_warnings"` - Skipped int `json:"skipped"` - Errors int `json:"errors"` - Phase string `json:"phase,omitempty"` - Message string `json:"message,omitempty"` - CurrentScope string `json:"current_scope,omitempty"` - TotalFiles int `json:"total_files,omitempty"` - FilesDiscovered int `json:"files_discovered,omitempty"` - FilesProcessed int `json:"files_processed,omitempty"` + New int `json:"new"` + Updated int `json:"updated"` + Unchanged int `json:"unchanged"` + Missing int `json:"missing"` + // MissingSkippedProtected counts files a scan declined to mark missing + // because their root was offline or unreadable. A non-zero value means the + // library is partly unavailable, not partly gone — without it a scan that + // covered for absent storage reports as an all-zero no-op. + MissingSkippedProtected int `json:"missing_skipped_protected"` + FilesDeleted int `json:"files_deleted"` + MembershipsRemoved int `json:"memberships_removed"` + ItemsDeleted int `json:"items_deleted"` + MatchedFiles int `json:"matched_files"` + RetriedItems int `json:"retried_items"` + StillUnmatchedWarnings int `json:"still_unmatched_warnings"` + Skipped int `json:"skipped"` + Errors int `json:"errors"` + Phase string `json:"phase,omitempty"` + Message string `json:"message,omitempty"` + CurrentScope string `json:"current_scope,omitempty"` + TotalFiles int `json:"total_files,omitempty"` + FilesDiscovered int `json:"files_discovered,omitempty"` + FilesProcessed int `json:"files_processed,omitempty"` } type ScanRun struct { diff --git a/internal/scanner/dead_root_test.go b/internal/scanner/dead_root_test.go index af7b5d3b..c97fd48f 100644 --- a/internal/scanner/dead_root_test.go +++ b/internal/scanner/dead_root_test.go @@ -193,12 +193,12 @@ func TestDeleteMissingByFolderProtectedRoots(t *testing.T) { // TestScanFolderDeadRootProtection walks the real scan pipeline end to end // with two on-disk roots and verifies the full dead-root story: // -// 1. a root that disappears has its files marked missing but never -// hard-deleted, even with trash emptying enabled and a zero grace -// (which would delete them in the very same scan without protection), -// and the folder surfaces a dead_root scan warning naming the root; -// 2. when the root comes back the rows resurrect in place (same ids, -// missing_since cleared) and the warning clears; +// 1. a root that disappears leaves its files completely untouched — neither +// marked missing nor hard-deleted, even with trash emptying enabled and a +// zero grace (which would delete them in the very same scan without +// protection) — and the folder surfaces a dead_root scan warning naming +// the root; +// 2. when the root comes back the rows are still live and the warning clears; // 3. deleting a file under a reachable root still purges its row after the // grace elapses (regression: the historical sweep is untouched). func TestScanFolderDeadRootProtection(t *testing.T) { @@ -278,9 +278,15 @@ func TestScanFolderDeadRootProtection(t *testing.T) { t.Fatalf("remove rootB: %v", err) } - // Scan 2: files under the dead root are marked missing (hidden) but the - // row must survive the sweep, and the folder must carry a dead_root - // warning naming the root. + // Scan 2: files under the dead root are left entirely alone — not marked + // missing, not deleted — and the folder carries a dead_root warning naming + // the root. + // + // Not marking them is the point: catalog reads filter on + // missing_since IS NULL, so a mark hides the title from browse, search and + // playback exactly as if it had been deleted. An unreachable root tells us + // nothing about whether its files exist, so hiding them turns a storage + // blip into a library outage that persists until the next good scan. result, err := scanner.ScanFolder(ctx, folder) if err != nil { t.Fatalf("scan 2: %v", err) @@ -288,6 +294,9 @@ func TestScanFolderDeadRootProtection(t *testing.T) { if len(result.UnreachableRoots) != 1 || result.UnreachableRoots[0] != rootB { t.Fatalf("scan 2 UnreachableRoots = %v, want [%s]", result.UnreachableRoots, rootB) } + if result.MissingSkippedProtected != 1 { + t.Fatalf("scan 2 MissingSkippedProtected = %d, want 1", result.MissingSkippedProtected) + } if _, missingA, foundA = fileRow(fileA); !foundA || missingA != nil { t.Fatalf("after scan 2: fileA found=%v missing=%v, want present and not missing", foundA, missingA) } @@ -295,8 +304,8 @@ func TestScanFolderDeadRootProtection(t *testing.T) { if !foundB { t.Fatal("after scan 2: fileB row was hard-deleted; dead-root protection failed") } - if missingB == nil { - t.Fatal("after scan 2: fileB not marked missing; it should be hidden") + if missingB != nil { + t.Fatalf("after scan 2: fileB marked missing at %v; an unreachable root must not hide its files", missingB) } if gotIDB != idB { t.Fatalf("after scan 2: fileB id changed %d -> %d", idB, gotIDB) @@ -317,8 +326,9 @@ func TestScanFolderDeadRootProtection(t *testing.T) { t.Fatal("after scan 2b: fileB row was hard-deleted on rescan") } - // Root B returns: the same row resurrects (same id, missing cleared) and - // the warning clears. + // Root B returns: the row is still the original, still live, and the + // warning clears. Because the outage never marked it missing, "recovery" + // is a no-op on the row rather than an un-hide. writeMovie(fileB) if _, err := scanner.ScanFolder(ctx, folder); err != nil { t.Fatalf("scan 3: %v", err) @@ -361,8 +371,8 @@ func TestScanFolderDeadRootProtection(t *testing.T) { // TestScanFolderNestedDeadChildRootProtection covers a child mount configured // INSIDE a reachable parent root (/parent plus /parent/child). Traversal // compaction drops the child, but it can die independently: its files must be -// protected from the sweep and the folder must warn, even though the parent -// scan is otherwise healthy. +// left untouched — neither hidden nor swept — and the folder must warn, even +// though the parent scan is otherwise healthy. func TestScanFolderNestedDeadChildRootProtection(t *testing.T) { pool := newDeadRootTestPool(t) ctx := context.Background() @@ -428,8 +438,12 @@ func TestScanFolderNestedDeadChildRootProtection(t *testing.T) { ).Scan(&gotID, &childMissing); err != nil { t.Fatalf("child row after scan 2 (was it hard-deleted?): %v", err) } - if childMissing == nil { - t.Fatal("child file not marked missing after its root died") + // The parent scan succeeded, but that says nothing about the dead child + // mount. Its rows must survive untouched — marking them missing on the + // parent's success would hide the child's whole catalog. + if childMissing != nil { + t.Fatalf("child file marked missing at %v after its root died; a dead child mount "+ + "must not be hidden just because its parent root scanned cleanly", childMissing) } if gotID != childID { t.Fatalf("child row id changed %d -> %d", childID, gotID) @@ -506,8 +520,17 @@ func TestScanFolderNestedSuspectEmptyChildRootProtection(t *testing.T) { ).Scan(&gotID, &missing); err != nil { t.Fatalf("child row after scan 2 (was it deleted?): %v", err) } - if gotID != childID || missing == nil { - t.Fatalf("child row id/missing = %d/%v, want %d/non-nil", gotID, missing, childID) + // The child mountpoint survived but its contents vanished with the mount, + // which is indistinguishable from an intentional emptying. The safe + // reading is to leave the rows visible: hiding them on a guess turns a + // dropped mount into a library outage, whereas an intentional emptying is + // confirmed explicitly through the cleanup allowance. + if gotID != childID { + t.Fatalf("child row id = %d, want %d", gotID, childID) + } + if missing != nil { + t.Fatalf("child row marked missing at %v; a suspect-empty child mount must not be "+ + "hidden just because its parent root scanned cleanly", missing) } } @@ -570,8 +593,9 @@ func TestScanFolderAllRootsDeadOutage(t *testing.T) { ).Scan(&missing); err != nil { t.Fatalf("file row after scan 2 (was it hard-deleted?): %v", err) } - if missing == nil { - t.Fatal("file not marked missing during all-roots-dead outage") + if missing != nil { + t.Fatalf("file marked missing at %v during an all-roots-dead outage; "+ + "a total outage must leave the catalog intact, not empty the library from users' view", missing) } var code *string @@ -719,8 +743,9 @@ func TestScanFolderSuspectEmptyRootProtection(t *testing.T) { if !foundB { t.Fatal("after scan 2: fileB row was hard-deleted; suspect-empty protection failed") } - if missingB == nil { - t.Fatal("after scan 2: fileB not marked missing; it should be hidden") + if missingB != nil { + t.Fatalf("after scan 2: fileB marked missing at %v; a suspect-empty root "+ + "is a lost mount, so its files must stay visible until the root is confirmed empty", missingB) } if gotIDB != idB { t.Fatalf("after scan 2: fileB id changed %d -> %d", idB, gotIDB) @@ -840,8 +865,9 @@ func TestScanFolderConfirmedCleanupPreservesDeadRoot(t *testing.T) { ).Scan(&missingA); err != nil { t.Fatalf("fileA row after scan 2 (was the dead root's catalog erased?): %v", err) } - if missingA == nil { - t.Fatal("fileA not marked missing during the outage") + if missingA != nil { + t.Fatalf("fileA marked missing at %v during the outage; confirming cleanup of a "+ + "reachable empty root must not disturb an unrelated unreachable root's files", missingA) } var countB int if err := pool.QueryRow(ctx, @@ -963,3 +989,667 @@ func TestSweepMissingAndReconcileProtectsDeadRootsFromScopedScans(t *testing.T) t.Fatal("genuinely deleted row under the reachable parent survived the sweep") } } + +// TestScanFolderFlappingRootNeverHidesPresentFiles reproduces the production +// failure this protection exists for: a CephFS-style mount that drops out and +// comes back while its files sit on disk the whole time. +// +// The file is never deleted and never changes. Only the mount flaps. Because +// every catalog read filters on missing_since IS NULL, a single spurious mark +// removes the title from browse, search and playback until the next successful +// scan — users experience it as "this file isn't available anymore" for media +// that is perfectly intact. The row must therefore come through every scan of +// the outage untouched. +func TestScanFolderFlappingRootNeverHidesPresentFiles(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Flapping Root Scan Test") + + base := t.TempDir() + live := filepath.Join(base, "live") + flappy := filepath.Join(base, "flappy") + liveFile := filepath.Join(live, "Alpha (2020)", "Alpha (2020).mkv") + flappyFile := filepath.Join(flappy, "Beta (2021)", "Beta (2021).mkv") + + write := func(path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + write(liveFile) + write(flappyFile) + + folder := &models.MediaFolder{ + ID: folderID, + Paths: []string{live, flappy}, + Type: "movies", + Name: "Flapping Root Scan Test", + Enabled: true, + } + // Trash emptying on with a zero grace: if a scan ever marks the file + // missing, the very next scan would also delete the row. + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + missingSince := func(path string) *time.Time { + t.Helper() + var missing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, path, + ).Scan(&missing); err != nil { + t.Fatalf("row for %s went away entirely: %v", path, err) + } + return missing + } + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("initial scan: %v", err) + } + if m := missingSince(flappyFile); m != nil { + t.Fatalf("baseline: file marked missing at %v", m) + } + + // Simulate the mount dropping and returning repeatedly. The payload on + // disk is restored byte-for-byte each cycle, exactly as a real mount + // returning exposes the same inodes. + stashed := filepath.Join(t.TempDir(), "stash") + for cycle := range 3 { + if err := os.Rename(flappy, stashed); err != nil { + t.Fatalf("cycle %d: drop mount: %v", cycle, err) + } + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("cycle %d: scan during outage: %v", cycle, err) + } + if result.MissingSkippedProtected != 1 { + t.Fatalf("cycle %d: MissingSkippedProtected = %d, want 1", + cycle, result.MissingSkippedProtected) + } + if m := missingSince(flappyFile); m != nil { + t.Fatalf("cycle %d: present file hidden at %v during a mount outage", cycle, m) + } + + if err := os.Rename(stashed, flappy); err != nil { + t.Fatalf("cycle %d: restore mount: %v", cycle, err) + } + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("cycle %d: scan after recovery: %v", cycle, err) + } + if m := missingSince(flappyFile); m != nil { + t.Fatalf("cycle %d: file hidden at %v after the mount returned", cycle, m) + } + } + + // The healthy sibling root must be unaffected throughout. + if m := missingSince(liveFile); m != nil { + t.Fatalf("file under the always-healthy root marked missing at %v", m) + } + + // And the genuine-deletion path still works: remove the file for real + // while its root is reachable, and the row is marked and swept. + if err := os.Remove(flappyFile); err != nil { + t.Fatalf("remove flappyFile: %v", err) + } + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan after real deletion: %v", err) + } + var stillThere bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM media_files WHERE media_folder_id = $1 AND file_path = $2)`, + folderID, flappyFile, + ).Scan(&stillThere); err != nil { + t.Fatalf("existence check: %v", err) + } + if stillThere { + t.Fatal("a genuinely deleted file under a reachable root survived; real deletions must still be detected") + } +} + +// TestScanFolderFirstScanAfterMountDropsProtectsLiveRows covers Codex review +// finding #2 on PR #472: suspect-empty detection used to require a root whose +// rows were ALL already missing, which meant it could only recognise a lost +// mount one scan too late. +// +// Here the mount drops leaving a reachable but empty mountpoint, and the rows +// are still live because nothing has marked them yet — the state on the very +// first scan after a real mount failure. The root must be classified suspect +// and its rows protected on that first scan, not after they have been hidden. +func TestScanFolderFirstScanAfterMountDropsProtectsLiveRows(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "First Outage Scan Test") + + base := t.TempDir() + live := filepath.Join(base, "live") + dropped := filepath.Join(base, "dropped") + liveFile := filepath.Join(live, "Alpha (2020)", "Alpha (2020).mkv") + droppedFile := filepath.Join(dropped, "Beta (2021)", "Beta (2021).mkv") + + write := func(path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + } + write(liveFile) + write(droppedFile) + + folder := &models.MediaFolder{ + ID: folderID, Paths: []string{live, dropped}, Type: "movies", + Name: "First Outage Scan Test", Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("baseline scan: %v", err) + } + var missing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, droppedFile).Scan(&missing); err != nil { + t.Fatalf("baseline row: %v", err) + } + if missing != nil { + t.Fatalf("baseline: row already missing at %v", missing) + } + + // The mount drops: contents vanish, the mountpoint directory remains and + // still probes reachable. The rows under it are all still live. + if err := os.RemoveAll(filepath.Join(dropped, "Beta (2021)")); err != nil { + t.Fatalf("empty the dropped root: %v", err) + } + + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("first outage scan: %v", err) + } + if len(result.SuspectEmptyRoots) != 1 || result.SuspectEmptyRoots[0] != dropped { + t.Fatalf("SuspectEmptyRoots = %v, want [%s] on the FIRST scan after the drop", + result.SuspectEmptyRoots, dropped) + } + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, droppedFile).Scan(&missing); err != nil { + t.Fatalf("row after outage scan (hard-deleted?): %v", err) + } + if missing != nil { + t.Fatalf("first scan after the mount dropped hid the row at %v; suspect-empty "+ + "protection must engage before the rows are marked, not after", missing) + } +} + +// TestCollectLogicalFilePathsReportsUnreadableEntries covers Codex review +// finding #3 on PR #472: the video walk swallowed per-entry read failures and +// reported no signal, so a mount dying partway through traversal produced a +// short file list indistinguishable from a large deletion. +func TestCollectLogicalFilePathsReportsUnreadableEntries(t *testing.T) { + t.Parallel() + + if os.Geteuid() == 0 { + t.Skip("running as root: permission bits do not deny access") + } + + root := t.TempDir() + readable := filepath.Join(root, "Alpha (2020)") + if err := os.MkdirAll(readable, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(readable, "Alpha (2020).mkv"), []byte("x"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + + // A subtree the walk cannot read stands in for the portion of a tree that + // becomes unreachable when a mount dies mid-traversal. + blocked := filepath.Join(root, "Beta (2021)") + if err := os.MkdirAll(blocked, 0o755); err != nil { + t.Fatalf("mkdir blocked: %v", err) + } + if err := os.WriteFile(filepath.Join(blocked, "Beta (2021).mkv"), []byte("x"), 0o644); err != nil { + t.Fatalf("write blocked: %v", err) + } + if err := os.Chmod(blocked, 0o000); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(blocked, 0o755) }) + + files, walkFailures, err := collectLogicalFilePaths(context.Background(), []string{root}, "movies") + if err != nil { + t.Fatalf("collectLogicalFilePaths: %v", err) + } + if len(walkFailures) != 1 { + t.Fatalf("walkFailures = %v, want exactly the unreadable subtree; a partial listing "+ + "would otherwise be treated as an authoritative inventory", walkFailures) + } + // The failure is scoped to the subtree that could not be read, not to the + // library root — otherwise one permanently broken entry would suppress + // missing-file reconciliation for the whole root on every future scan. + if walkFailures[0] != blocked { + t.Fatalf("walkFailures[0] = %q, want the blocked subtree %q", walkFailures[0], blocked) + } + if walkFailures[0] == root { + t.Fatal("failure was recorded against the library root; protection must be scoped to the subtree") + } + // The readable file is still found — one bad subtree must not abort the walk. + if len(files) != 1 { + t.Fatalf("files = %v, want just the readable one", files) + } +} + +// TestScanFolderBrokenSymlinkDoesNotFreezeRootReconciliation covers Codex +// review finding #3 on the follow-up commit: walk failures were counted, not +// located, so any failure protected the entire library root. +// +// A dangling symlink is both common and permanent, so that would have +// suppressed missing-file reconciliation for its whole root on every future +// scan — genuinely deleted titles would stay live forever. The failure must be +// scoped to the offending path so the rest of the root still reconciles. +func TestScanFolderBrokenSymlinkDoesNotFreezeRootReconciliation(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Broken Symlink Scan Test") + + root := t.TempDir() + keeper := filepath.Join(root, "Keeper (2020)", "Keeper (2020).mkv") + doomed := filepath.Join(root, "Doomed (2021)", "Doomed (2021).mkv") + for _, p := range []string{keeper, doomed} { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(p, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + } + // A permanently dangling symlink, the everyday case. + if err := os.Symlink(filepath.Join(root, "nowhere"), filepath.Join(root, "dangling.mkv")); err != nil { + t.Skipf("symlinks not supported: %v", err) + } + + folder := &models.MediaFolder{ + ID: folderID, Paths: []string{root}, Type: "movies", + Name: "Broken Symlink Scan Test", Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("baseline scan: %v", err) + } + + // Genuinely delete one title. The dangling symlink is still there. + if err := os.RemoveAll(filepath.Dir(doomed)); err != nil { + t.Fatalf("remove doomed: %v", err) + } + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan after deletion: %v", err) + } + + var missing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, doomed).Scan(&missing); err != nil { + // The row may already have been swept (zero grace), which also counts + // as correctly reconciled. + if !strings.Contains(err.Error(), "no rows") { + t.Fatalf("doomed row: %v", err) + } + missing = nil + } else if missing == nil { + t.Fatal("a genuinely deleted title stayed live because an unrelated dangling symlink " + + "marked the whole root unreconcilable") + } + + // The surviving title must be untouched throughout. + var keeperMissing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, keeper).Scan(&keeperMissing); err != nil { + t.Fatalf("keeper row: %v", err) + } + if keeperMissing != nil { + t.Fatalf("surviving title marked missing at %v", keeperMissing) + } +} + +// TestReprobeNestedRootsCatchesMidScanChildDrop covers Codex review finding #5 +// on PR #472: a configured child mount that is healthy at the initial probe +// but gone by the time its compacted parent is walked. +// +// Compaction folds the child into its parent, so it never gets a scope of its +// own, and the post-walk re-probe only revisits scopes that walked empty — +// never a populated parent. Without a re-probe the child's rows are marked +// missing on the parent's success, which is the mid-scan disconnect this +// protection exists for. +func TestReprobeNestedRootsCatchesMidScanChildDrop(t *testing.T) { + + base := t.TempDir() + parent := filepath.Join(base, "media") + child := filepath.Join(parent, "child-mount") + sibling := filepath.Join(base, "unrelated") + for _, d := range []string{parent, child, sibling} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + // The child holds media, so it is a live mount rather than a bare + // mountpoint while it is healthy. + if err := os.WriteFile(filepath.Join(child, "Alpha (2020).mkv"), []byte("x"), 0o644); err != nil { + t.Fatalf("seed child media: %v", err) + } + configured := []string{parent, child, sibling} + + pool := newDeadRootTestPool(t) + s := &Scanner{fileRepo: NewFileRepository(pool)} + + // Healthy child: nothing to protect. + unreachable, suspect, err := s.reprobeNestedRoots(context.Background(), 1, configured, parent, false) + if err != nil { + t.Fatalf("reprobeNestedRoots (healthy): %v", err) + } + if len(unreachable) != 0 || len(suspect) != 0 { + t.Fatalf("unreachable=%v suspect=%v, want none while the child is reachable", unreachable, suspect) + } + + // The child mount drops mid-scan. + if err := os.RemoveAll(child); err != nil { + t.Fatalf("drop child: %v", err) + } + unreachable, suspect, err = s.reprobeNestedRoots(context.Background(), 1, configured, parent, false) + if err != nil { + t.Fatalf("reprobeNestedRoots (dropped): %v", err) + } + // It must land in the UNREACHABLE bucket specifically: reporting a dropped + // mount as suspect-empty would hand an operator the wrong diagnosis. + if len(unreachable) != 1 || unreachable[0] != child { + t.Fatalf("unreachable = %v, want [%s]: a child that drops after the initial probe "+ + "must be caught before its parent's scope is reconciled", unreachable, child) + } + if len(suspect) != 0 { + t.Fatalf("suspect = %v, want none: an unreachable child is not suspect-empty", suspect) + } + + // Only roots nested under this parent are considered — a sibling root has + // its own scope and must not be swept in here. + unreachable, suspect, err = s.reprobeNestedRoots(context.Background(), 1, configured, sibling, false) + if err != nil { + t.Fatalf("reprobeNestedRoots (sibling): %v", err) + } + if len(unreachable) != 0 || len(suspect) != 0 { + t.Fatalf("unreachable=%v suspect=%v, want none: %s has no nested configured roots", + unreachable, suspect, sibling) + } +} + +// TestScanFolderProtectedChildRootSurvivesTrashSweep asserts that a nested +// child root which is offline at scan time keeps its already-missing rows +// through the folder-wide trash sweep, even when those rows are long past the +// removal grace. +// +// Scope note: this stages the child as unreachable BEFORE the scan, so the +// initial probe classifies it and the protection comes from that path. It does +// NOT reproduce the mid-scan drop behind Codex finding #6 — where the child is +// healthy at probe time and dies during the walk, so only reprobeNestedRoots +// sees it. Staging that race needs the drop to land between the probe and the +// walk, which is not reachable from a test without hooks. The fix for that +// path (carrying reprobedRoots into protectedScanRoots) is therefore covered +// by inspection, not by this test; what this test does pin is that the sweep +// honours the protected set it is given. +// +// Trash emptying is on with a zero grace, so any row left unprotected is +// deleted immediately rather than merely hidden. +func TestScanFolderProtectedChildRootSurvivesTrashSweep(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Mid-Scan Drop Sweep Test") + + base := t.TempDir() + parent := filepath.Join(base, "media") + child := filepath.Join(parent, "child-mount") + parentFile := filepath.Join(parent, "Keeper (2020)", "Keeper (2020).mkv") + childFile := filepath.Join(child, "Child (2021)", "Child (2021).mkv") + for _, p := range []string{parentFile, childFile} { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(p, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + } + + folder := &models.MediaFolder{ + ID: folderID, Paths: []string{parent, child}, Type: "movies", + Name: "Mid-Scan Drop Sweep Test", Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("baseline scan: %v", err) + } + var childID int + if err := pool.QueryRow(ctx, + `SELECT id FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, childFile).Scan(&childID); err != nil { + t.Fatalf("child row: %v", err) + } + + // Put the child's row in the state the sweep would delete: already marked + // missing, well past the (zero) removal grace. + if _, err := pool.Exec(ctx, + `UPDATE media_files SET missing_since = NOW() - INTERVAL '48 hours' WHERE id = $1`, + childID); err != nil { + t.Fatalf("pre-mark child row: %v", err) + } + + // The child mount drops. The parent stays healthy and still walks files, + // so the scan takes the populated-parent path. + if err := os.RemoveAll(child); err != nil { + t.Fatalf("drop child mount: %v", err) + } + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan after child drop: %v", err) + } + + var survives bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM media_files WHERE id = $1)`, childID).Scan(&survives); err != nil { + t.Fatalf("existence check: %v", err) + } + if !survives { + t.Fatal("row under an offline child root was hard-deleted by the trash sweep; " + + "an outage must never be a trigger for permanent deletion") + } + + // The parent's own file must be unaffected throughout. + var parentMissing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, parentFile).Scan(&parentMissing); err != nil { + t.Fatalf("parent row: %v", err) + } + if parentMissing != nil { + t.Fatalf("healthy parent file marked missing at %v", parentMissing) + } +} + +// TestScanFolderUnreadableSubtreeSurvivesTrashSweep covers Codex review +// finding #8 on PR #472 — the sibling of finding #6, and the second data-loss +// path in this area. +// +// applyScopedScan protected rows under an unreadable directory locally via +// scope.walkFailures, but the folder-wide protected set was rebuilt without +// those paths. With trash emptying on, DeleteMissingByFolder could then +// permanently delete rows past the removal grace beneath a directory this scan +// could not read — deleting on the strength of an observation never made. +// +// Both that fix and #6's now flow through one accumulated protected set, so +// this test guards the propagation rather than one symptom of losing it. +func TestScanFolderUnreadableSubtreeSurvivesTrashSweep(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: permission bits do not deny access") + } + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Unreadable Subtree Sweep Test") + + root := t.TempDir() + keeper := filepath.Join(root, "Keeper (2020)", "Keeper (2020).mkv") + hidden := filepath.Join(root, "Locked (2021)", "Locked (2021).mkv") + for _, p := range []string{keeper, hidden} { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(p, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + } + + folder := &models.MediaFolder{ + ID: folderID, Paths: []string{root}, Type: "movies", + Name: "Unreadable Subtree Sweep Test", Enabled: true, + } + // Trash emptying on, zero grace: anything left unprotected and already + // marked missing is deleted on sight. + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("baseline scan: %v", err) + } + var hiddenID int + if err := pool.QueryRow(ctx, + `SELECT id FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, hidden).Scan(&hiddenID); err != nil { + t.Fatalf("hidden row: %v", err) + } + // Put it in the state the sweep would delete. + if _, err := pool.Exec(ctx, + `UPDATE media_files SET missing_since = NOW() - INTERVAL '48 hours' WHERE id = $1`, + hiddenID); err != nil { + t.Fatalf("pre-mark: %v", err) + } + + // The subtree becomes unreadable — a permission fault, or storage that + // stopped answering for part of the tree. + lockedDir := filepath.Dir(hidden) + if err := os.Chmod(lockedDir, 0o000); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(lockedDir, 0o755) }) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan with unreadable subtree: %v", err) + } + + var survives bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM media_files WHERE id = $1)`, hiddenID).Scan(&survives); err != nil { + t.Fatalf("existence check: %v", err) + } + if !survives { + t.Fatal("row beneath an unreadable directory was hard-deleted by the trash sweep; " + + "a scan must not delete on the strength of an observation it could not make") + } + + // The readable title is unaffected. + var keeperMissing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, keeper).Scan(&keeperMissing); err != nil { + t.Fatalf("keeper row: %v", err) + } + if keeperMissing != nil { + t.Fatalf("readable title marked missing at %v", keeperMissing) + } +} + +// TestScanFolderEmptiedNestedChildUnderEmptyParentStaysVisible pins that a +// nested child whose contents vanish keeps its rows visible, in the awkward +// topology where the child holds the parent's only media and a healthy sibling +// root keeps the folder-wide empty guard from firing. +// +// Scope note, stated because it is easy to misread: the child is emptied +// BEFORE this scan, so the INITIAL probe classifies it as suspect-empty and +// protection arrives through that path. It therefore does NOT exercise the +// pending-scope re-probe added for Codex finding #11, which only matters when +// the child drops AFTER the initial probe. Verified: this test still passes +// with that re-probe disabled. +// +// Staging the real mid-scan race needs the drop to land between the probe and +// the walk, which a test cannot reach without hooks. That fix — like the +// populated-scope re-probe before it — rests on inspection, not on this test. +// What this test does guard is the end-to-end outcome for the topology, which +// no other test covers. +func TestScanFolderEmptiedNestedChildUnderEmptyParentStaysVisible(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Pending Empty Parent Test") + + base := t.TempDir() + parent := filepath.Join(base, "parent") + child := filepath.Join(parent, "child-mount") + sibling := filepath.Join(base, "sibling") + // The parent's ONLY media is inside the child. + childFile := filepath.Join(child, "Child (2021)", "Child (2021).mkv") + siblingFile := filepath.Join(sibling, "Sibling (2020)", "Sibling (2020).mkv") + for _, p := range []string{childFile, siblingFile} { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(p, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + } + + folder := &models.MediaFolder{ + ID: folderID, Paths: []string{parent, child, sibling}, Type: "movies", + Name: "Pending Empty Parent Test", Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("baseline scan: %v", err) + } + var childID int + if err := pool.QueryRow(ctx, + `SELECT id FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, childFile).Scan(&childID); err != nil { + t.Fatalf("child row: %v", err) + } + + // The child mount's contents vanish but its mountpoint directory remains, + // so the parent still looks present and non-empty from above. + if err := os.RemoveAll(filepath.Join(child, "Child (2021)")); err != nil { + t.Fatalf("empty child mount: %v", err) + } + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan after child emptied: %v", err) + } + + var missing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND id = $2`, + folderID, childID).Scan(&missing); err != nil { + t.Fatalf("child row after scan (hard-deleted?): %v", err) + } + if missing != nil { + t.Fatalf("child row hidden at %v: an emptied nested child must stay visible when its "+ + "parent also walks empty and a sibling root keeps the folder guard from firing", missing) + } + + // The healthy sibling is untouched. + var siblingMissing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, siblingFile).Scan(&siblingMissing); err != nil { + t.Fatalf("sibling row: %v", err) + } + if siblingMissing != nil { + t.Fatalf("healthy sibling marked missing at %v", siblingMissing) + } +} diff --git a/internal/scanner/ebook_scan.go b/internal/scanner/ebook_scan.go index 3c96d002..4cafe6b9 100644 --- a/internal/scanner/ebook_scan.go +++ b/internal/scanner/ebook_scan.go @@ -53,9 +53,9 @@ func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolde type ebookRootScan struct { root string files []string - fileOnly bool // explicit single-file scan; index it but never reconcile a subtree - rootErr error // root stat failed, or the root is not a directory - walkFailures int // entries within the subtree the walk could not read or resolve + fileOnly bool // explicit single-file scan; index it but never reconcile a subtree + rootErr error // root stat failed, or the root is not a directory + walkFailures []string // logical paths within the subtree the walk could not read or resolve } // failed reports whether the walk under this root is known to be incomplete. @@ -64,7 +64,7 @@ type ebookRootScan struct { // a mid-walk subtree error) must not cascade into marking — and, with // empty_trash_after_scan, deleting — everything under it. func (r *ebookRootScan) failed() bool { - return r.rootErr != nil || r.walkFailures > 0 + return r.rootErr != nil || len(r.walkFailures) > 0 } // collectEbookRootScans walks every configured root with the shared @@ -104,7 +104,7 @@ func collectEbookRootScans(ctx context.Context, folderID int, roots []string) ([ slog.WarnContext(ctx, "ebook scan: root walk incomplete; root excluded from missing-file reconciliation", "component", "scanner", "folder_id", folderID, "root", cleanRoot, - "walk_failures", scan.walkFailures, + "walk_failures", len(scan.walkFailures), "error", scan.rootErr, ) } diff --git a/internal/scanner/ebook_test.go b/internal/scanner/ebook_test.go index a6e538dd..2af2a572 100644 --- a/internal/scanner/ebook_test.go +++ b/internal/scanner/ebook_test.go @@ -1611,7 +1611,7 @@ func TestCollectEbookRootScansMidWalkSubtreeErrorExcludesRoot(t *testing.T) { if len(scans) != 1 { t.Fatalf("scans = %d, want 1", len(scans)) } - if scans[0].walkFailures == 0 || !scans[0].failed() { + if len(scans[0].walkFailures) == 0 || !scans[0].failed() { t.Fatalf("scan = %+v, want walk failure recorded for unreadable subtree", scans[0]) } if len(scans[0].files) != 1 { diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index cc46f0ab..264e05ec 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -2682,6 +2682,58 @@ func (r *FileRepository) DeleteMissingByFolder(ctx context.Context, folderID int return int(tag.RowsAffected()), nil } +// ListRootsWithCatalogedFiles returns the subset of roots (in input order) +// that still have any media_files rows at or under them in the folder, +// whether those rows are present or already marked missing. +// +// This is the proactive counterpart to ListRootsWithOnlyMissingFiles. That +// query requires a root to have NO live rows left, which means it can only +// recognise a lost mount after a scan has already marked its files missing — +// i.e. after the damage is done. For deciding whether to mark in the first +// place, the question is simply "does the catalog believe anything lives +// here", because an empty-but-reachable directory that still owns cataloged +// files is the signature of a dropped mount exposing its bare mountpoint. +// +// A genuinely emptied root also matches, which is intended: emptying a root +// is confirmed through the operator's one-time cleanup allowance rather than +// inferred from a single scan. +func (r *FileRepository) ListRootsWithCatalogedFiles(ctx context.Context, folderID int, roots []string) ([]string, error) { + if len(roots) == 0 { + return nil, nil + } + patterns := make([]string, len(roots)) + for i, root := range roots { + patterns[i] = pathscope.PrefixLike(root) + } + rows, err := r.pool.Query(ctx, ` + SELECT r.root + FROM unnest($2::text[], $3::text[]) WITH ORDINALITY AS r(root, pattern, ord) + WHERE EXISTS ( + SELECT 1 FROM media_files mf + WHERE mf.media_folder_id = $1 + AND (mf.file_path = r.root OR mf.file_path LIKE r.pattern ESCAPE '\') + ) + ORDER BY r.ord + `, folderID, roots, patterns) + if err != nil { + return nil, fmt.Errorf("querying roots with cataloged files: %w", err) + } + defer rows.Close() + + occupied := make([]string, 0) + for rows.Next() { + var root string + if err := rows.Scan(&root); err != nil { + return nil, fmt.Errorf("scanning root with cataloged files: %w", err) + } + occupied = append(occupied, root) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating roots with cataloged files: %w", err) + } + return occupied, nil +} + // ListRootsWithOnlyMissingFiles returns the subset of roots (in input order) // that still have media_files rows at or under them in the folder but none // that are present (missing_since IS NULL). A reachable root in this state is diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 03114d56..2077b38a 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -478,9 +478,17 @@ func isIgnoredDirectoryPath(path string) bool { // Callers that pass a non-nil counter (the book scanners) use it to exclude // incompletely walked roots from missing-file reconciliation; the video walk // passes nil and keeps its historical log-and-continue behavior. -func recordWalkFailure(failures *int) { +// recordWalkFailure notes the logical path the walk could not read or resolve. +// +// The path matters, not just the count: an unreadable directory hides whatever +// was inside it, but a dangling symlink hides nothing beyond itself. Recording +// where each failure happened lets callers protect exactly the affected +// subtree instead of the whole library root — otherwise a single permanently +// broken symlink would suppress missing-file reconciliation for that root on +// every future scan, and genuinely deleted titles would never be retired. +func recordWalkFailure(failures *[]string, path string) { if failures != nil { - *failures++ + *failures = append(*failures, path) } } @@ -491,7 +499,7 @@ func walkLogicalTree( mode walkMode, visitedPhysicalDirs map[string]struct{}, filePaths *[]string, - walkFailures *int, + walkFailures *[]string, ) error { if ctx != nil { if err := ctx.Err(); err != nil { @@ -502,7 +510,7 @@ func walkLogicalTree( info, err := os.Lstat(physicalPath) if err != nil { slog.WarnContext(ctx, "scanner: walk lstat failed", "component", "scanner", "path", logicalPath, "physical_path", physicalPath, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalPath) return nil } @@ -510,13 +518,13 @@ func walkLogicalTree( resolved, err := filepath.EvalSymlinks(physicalPath) if err != nil { slog.WarnContext(ctx, "scanner: symlink resolve failed", "component", "scanner", "path", logicalPath, "physical_path", physicalPath, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalPath) return nil } targetInfo, err := os.Stat(resolved) if err != nil { slog.WarnContext(ctx, "scanner: symlink stat failed", "component", "scanner", "path", logicalPath, "resolved_path", resolved, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalPath) return nil } if targetInfo.IsDir() { @@ -544,7 +552,7 @@ func walkLogicalTree( canonicalDir, err := canonicalWalkPath(physicalPath) if err != nil { slog.WarnContext(ctx, "scanner: canonical path resolution failed", "component", "scanner", "path", logicalPath, "physical_path", physicalPath, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalPath) return nil } if _, seen := visitedPhysicalDirs[canonicalDir]; seen { @@ -562,7 +570,7 @@ func walkLogicalTree( entries, err := os.ReadDir(physicalPath) if err != nil { slog.WarnContext(ctx, "scanner: directory read failed", "component", "scanner", "path", logicalPath, "physical_path", physicalPath, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalPath) return nil } for _, entry := range entries { @@ -579,13 +587,13 @@ func walkLogicalTree( resolved, err := filepath.EvalSymlinks(physicalChild) if err != nil { slog.WarnContext(ctx, "scanner: symlink resolve failed", "component", "scanner", "path", logicalChild, "physical_path", physicalChild, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalChild) continue } targetInfo, err := os.Stat(resolved) if err != nil { slog.WarnContext(ctx, "scanner: symlink stat failed", "component", "scanner", "path", logicalChild, "resolved_path", resolved, "error", err) - recordWalkFailure(walkFailures) + recordWalkFailure(walkFailures, logicalChild) continue } if targetInfo.IsDir() { @@ -621,27 +629,43 @@ func walkLogicalTree( return nil } -func collectLogicalFilePaths(ctx context.Context, walkRoots []string, libraryType string) ([]string, error) { +// collectLogicalFilePaths walks the given roots and returns the media files +// found, plus how many entries the walk could not read or resolve. +// +// walkLogicalTree deliberately swallows per-entry Lstat/ReadDir failures so one +// bad file cannot abort a scan of a million. That makes the returned list +// non-authoritative wherever a failure occurred: a mount that dies partway +// through traversal yields a short list that looks exactly like a large +// deletion. +// +// The second return value is the logical path of every entry the walk could +// not read or resolve. Callers must treat those paths as unreconcilable — +// anything cataloged at or under one of them may exist without having been +// seen. Scoping the protection to those paths rather than to the whole root +// matters: a dangling symlink is permanent, and protecting its entire library +// root would suppress missing-file reconciliation there on every future scan. +func collectLogicalFilePaths(ctx context.Context, walkRoots []string, libraryType string) ([]string, []string, error) { filePaths := make([]string, 0) visitedPhysicalDirs := make(map[string]struct{}) mode := walkModeFor(libraryType) + walkFailures := make([]string, 0) for _, rootPath := range walkRoots { if ctx != nil { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } } cleanRoot := filepath.Clean(rootPath) if cleanRoot == "" || cleanRoot == "." { continue } - if err := walkLogicalTree(ctx, cleanRoot, cleanRoot, mode, visitedPhysicalDirs, &filePaths, nil); err != nil { - return nil, err + if err := walkLogicalTree(ctx, cleanRoot, cleanRoot, mode, visitedPhysicalDirs, &filePaths, &walkFailures); err != nil { + return nil, nil, err } } - return filePaths, nil + return filePaths, walkFailures, nil } func (s *Scanner) scanPaths( @@ -693,7 +717,7 @@ func (s *Scanner) scanPaths( Message: "Discovering media files", CurrentScope: firstScope(reconcileRoots), }) - filePaths, walkErr := collectLogicalFilePaths(ctx, walkRoots, folder.Type) + filePaths, walkFailures, walkErr := collectLogicalFilePaths(ctx, walkRoots, folder.Type) if walkErr != nil { return nil, fmt.Errorf("walking media roots: %w", walkErr) } @@ -702,6 +726,9 @@ func (s *Scanner) scanPaths( "scope", firstScope(reconcileRoots), "files", len(filePaths), ) + if len(walkFailures) > 0 { + logIncompleteWalk(ctx, folder.ID, reconcileRoots, walkFailures) + } reportProgress(ctx, ProgressUpdate{ Phase: "processing", Message: "Processing discovered files", @@ -735,11 +762,33 @@ func (s *Scanner) scanPaths( applyGroupOverrides(&groupInference, groupOverrides) result.RootObservations = rootInference.Observations s.logRootInferenceDisagreements(rootInference.Assignments) + + // Probe before any reconciliation, not just before missing-marking. + // Pruning deletes snapshots and observed/group locations the walk did not + // see, so it is unsound for the same reasons marking is: a suspect-empty + // mountpoint walks clean and reports no failures, and pruning against that + // empty inventory strips metadata for media that is still there. Deciding + // protection after these calls preserved the media rows while their + // snapshots had already been deleted. + protectedRoots, err := s.protectedConfiguredRoots(ctx, folder) + if err != nil { + return nil, err + } + // Wherever the walk could not read, its file list is a lower bound rather + // than an inventory, so anything cataloged under those paths must not be + // reconciled. Only those paths are protected — a dangling symlink must not + // exempt its whole library root forever. + if len(walkFailures) > 0 { + protectedRoots = append(append([]string(nil), protectedRoots...), walkFailures...) + } + pruneUnseen := len(walkFailures) == 0 && !anyPathWithinRoots(protectedRoots, reconcileRoots) + if err := s.reconcileScannedRoots( ctx, folder.ID, reconcileRoots, rootInference.Snapshots, + pruneUnseen, ); err != nil { return nil, fmt.Errorf("reconciling scanned roots: %w", err) } @@ -885,41 +934,19 @@ func (s *Scanner) scanPaths( return nil, fmt.Errorf("syncing present library state for folder %d: %w", folder.ID, err) } - if err := s.reconcileScannedGroups(ctx, folder.ID, allowEmptyRootGuard, reconcileRoots, !allowEmptyRootGuard, groupInference); err != nil { + // Group pruning follows the same rule as snapshot pruning above. + if err := s.reconcileScannedGroups(ctx, folder.ID, allowEmptyRootGuard, reconcileRoots, + !allowEmptyRootGuard && pruneUnseen, groupInference); err != nil { return nil, fmt.Errorf("reconciling scanned groups: %w", err) } - // Mark files that were in the DB but not found on disk as missing. - now := time.Now().UTC() - for _, existing := range existingFiles { - if seenPaths[existing.FilePath] { - continue - } - // Only mark as missing if not already marked. - if existing.MissingSince == nil { - if err := s.fileRepo.MarkMissing(ctx, existing.ID, now); err != nil { - slog.ErrorContext(ctx, "scanner: failed to mark file missing", "component", "scanner", - "path", existing.FilePath, - "error", err, - ) - result.Errors++ - continue - } - } - result.Missing++ - } + // protectedRoots was resolved above, before any reconciliation ran. + s.markMissingExcludingProtected(ctx, folder.ID, existingFiles, seenPaths, protectedRoots, result) // Empty trash: delete files marked as missing for longer than the removal // grace for this folder. Safe because the empty-root guard (above) returns // early when 0 files are found on disk, so we only reach here when the - // root is populated. The sweep is folder-wide, so a scoped scan of a - // healthy subtree must not hard-delete rows an unreachable or - // suspect-empty sibling root left marked missing: probe the configured - // library roots and exempt any that are currently protected. - protectedRoots, err := s.protectedConfiguredRoots(ctx, folder) - if err != nil { - return nil, err - } + // root is populated. removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots) if err != nil { return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) @@ -1019,6 +1046,12 @@ type scopedScan struct { rootInference rootInferenceResult groupInference groupInferenceResult result *ScanResult + // walkFailures holds the logical paths this scope's walk could not read or + // resolve. Anything cataloged at or under one of them may exist without + // having been seen, so those paths are excluded from missing + // reconciliation — and their presence also suppresses snapshot/group + // pruning, which would otherwise prune against a partial inventory. + walkFailures []string } func (s *Scanner) scanFolderByRoots( @@ -1035,12 +1068,12 @@ func (s *Scanner) scanFolderByRoots( roots := compactScanRoots(configuredRoots) // An unreachable root (dead drive, lost mount) is temporarily offline, not - // removed from the library: its files are still marked missing below so - // clients stop seeing them, but nothing under it may be hard-deleted while - // it stays unreachable — trash emptying and item purging are guarded - // against these roots further down. Probe every CONFIGURED path, not the - // compacted traversal roots: a nested child mount is dropped by compaction - // but can die independently of its reachable parent. + // removed from the library: its files are left untouched below — not + // marked missing, not trashed, not purged — for as long as it stays + // unreachable. Marking would be enough to hide them, since every catalog + // read filters on missing_since IS NULL. Probe every CONFIGURED path, not + // the compacted traversal roots: a nested child mount is dropped by + // compaction but can die independently of its reachable parent. configuredProbes := rootcheck.ProbeManyWithTimeout(ctx, configuredRoots, rootcheck.DefaultProbeTimeout) unreachableRoots := make([]string, 0) suspectRoots := make([]string, 0) @@ -1068,6 +1101,33 @@ func (s *Scanner) scanFolderByRoots( unreachableSet[root] = true } + // Whether the operator has armed the one-time cleanup allowance decides + // how a suspect-empty NESTED child is treated in the walked-parent branch + // below. That branch runs before the allowance is consumed, and a scope it + // has already reconciled cannot be revisited — so without checking here, + // arming the allowance would consume the confirmation while the child's + // rows stayed protected indefinitely. Read it without consuming; the + // consume still happens in its existing places. + cleanupArmed, err := s.emptyCleanupArmed(ctx, folder.ID) + if err != nil { + return nil, err + } + + // Protection discovered after the initial probe must reach every later + // destructive step, not just the scope that found it. Repeatedly missing + // one of these propagation edges is what produced several defects in this + // area, so each source accumulates folder-wide here and every consumer + // reads the combined set via protectedScanRoots below. + // + // Unreachable and suspect are tracked apart because they mean different + // things to an operator and are reported separately. + reprobedUnreachable := make([]string, 0) + reprobedSuspect := make([]string, 0) + // unreadablePaths are paths some scope's walk could not read. Rows beneath + // them were never verified this scan, so they must survive the sweep just + // as an offline root's do. + unreadablePaths := make([]string, 0) + pendingEmptyScopes := make([]*scopedScan, 0) totalExisting := 0 seenAnyFiles := false @@ -1082,7 +1142,9 @@ func (s *Scanner) scanFolderByRoots( scopeWalkRoots := []string{root} if unreachableSet[root] { // Skip walking a dead root — it would yield nothing anyway. Its - // scope still reconciles below so existing rows are marked missing. + // scope still reconciles below (roots, groups, memberships), but + // its existing rows are protected from missing-marking rather + // than swept up by the empty walk. scopeWalkRoots = nil } scope, err := s.scanScope(ctx, folder, scopeWalkRoots, []string{root}) @@ -1099,7 +1161,51 @@ func (s *Scanner) scanFolderByRoots( if len(scope.filePaths) > 0 { seenAnyFiles = true beforeErrors := scope.result.Errors - if err := s.applyScopedScan(ctx, folder, scope, false, nil); err != nil { + // Pass both protected sets even though this scope walked files: a + // nested child mount compacted into this root can be dead, or can + // be a suspect-empty mountpoint, and either way its rows fall + // inside this scope's existing files. Without them the child's + // catalog gets marked missing on the parent's success. + // + // Suspect-empty children are protected unless the operator has + // explicitly confirmed cleanup — that confirmation is the + // deliberate way to retire an emptied root, and honouring it here + // is what stops the allowance being consumed to no effect. + // Unreachable roots are protected either way: an outage is never + // a confirmation to erase a root's catalog. + scopeProtected := append([]string(nil), unreachableRoots...) + if !cleanupArmed { + scopeProtected = append(scopeProtected, suspectRoots...) + } + // unreachableRoots/suspectRoots were sampled before the walk. A + // nested child can drop in between — healthy at probe time, gone + // by the time its parent is walked — and the post-walk re-probe + // below only revisits scopes that walked empty, never a populated + // parent like this one. Re-probe this root's configured children + // now so a mid-scan disconnect is caught before its rows are + // reconciled. + freshUnreachable, freshSuspect, err := s.reprobeNestedRoots(ctx, folder.ID, configuredRoots, root, cleanupArmed) + if err != nil { + return nil, err + } + scopeProtected = append(scopeProtected, freshUnreachable...) + scopeProtected = append(scopeProtected, freshSuspect...) + // A root discovered offline here must stay protected for the rest + // of the scan, not just for this scope: the folder-wide membership + // reconcile and trash sweep below run off these sets too. Without + // that, rows under a child that dropped mid-scan and are already + // past the removal grace get hard-deleted by the very scan that + // noticed the outage. + for _, protectedRoot := range freshUnreachable { + reprobedUnreachable = appendUniquePath(reprobedUnreachable, protectedRoot) + } + for _, protectedRoot := range freshSuspect { + reprobedSuspect = appendUniquePath(reprobedSuspect, protectedRoot) + } + for _, unreadable := range scope.walkFailures { + unreadablePaths = appendUniquePath(unreadablePaths, unreadable) + } + if err := s.applyScopedScan(ctx, folder, scope, false, scopeProtected); err != nil { return nil, err } mergeCleanupResult(result, scope.result, beforeErrors) @@ -1109,6 +1215,9 @@ func (s *Scanner) scanFolderByRoots( // Empty scopes stay pending until after the loop: whether they may // force-delete (confirmed cleanup) and whether their roots must be // treated as suspect depends on what the other roots produced. + for _, unreadable := range scope.walkFailures { + unreadablePaths = appendUniquePath(unreadablePaths, unreadable) + } pendingEmptyScopes = append(pendingEmptyScopes, scope) } @@ -1120,6 +1229,22 @@ func (s *Scanner) scanFolderByRoots( if root == "" || unreachableSet[root] || len(pending.existingFiles) == 0 { continue } + // Re-probe this scope's nested configured children too. A parent whose + // only media lived in a child walks empty when that child drops, and + // probing the parent alone says nothing — it still contains the + // child's bare mountpoint directory, so it reads as present and + // non-empty. Without this the child is never classified and its rows + // are marked or swept. The populated-scope branch does the same. + nestedUnreachable, nestedSuspect, nerr := s.reprobeNestedRoots(ctx, folder.ID, configuredRoots, root, cleanupArmed) + if nerr != nil { + return nil, nerr + } + for _, protectedRoot := range nestedUnreachable { + reprobedUnreachable = appendUniquePath(reprobedUnreachable, protectedRoot) + } + for _, protectedRoot := range nestedSuspect { + reprobedSuspect = appendUniquePath(reprobedSuspect, protectedRoot) + } probe := rootcheck.ProbeWithTimeout(ctx, root, rootcheck.DefaultProbeTimeout) if !probe.Reachable { logUnreachableRoot(ctx, folder.ID, root, probe) @@ -1180,9 +1305,30 @@ func (s *Scanner) scanFolderByRoots( if allowEmptyCleanup { suspectRoots = nil } + // A root that dropped mid-scan is as much an outage as one that failed the + // initial probe; report it so the folder warning and scan result do not + // present a partial scan as clean. Each keeps its own classification — + // reporting a suspect-empty child as unreachable would hand an operator + // contradictory failure information. + for _, protectedRoot := range reprobedUnreachable { + unreachableRoots = appendUniquePath(unreachableRoots, protectedRoot) + } + if !allowEmptyCleanup { + for _, protectedRoot := range reprobedSuspect { + suspectRoots = appendUniquePath(suspectRoots, protectedRoot) + } + } + result.UnreachableRoots = unreachableRoots result.SuspectEmptyRoots = suspectRoots + // The single protected set every later destructive step reads: offline + // roots, suspect-empty roots, and paths no walk could read. Anything that + // discovers protection must land here, or the trash sweep will delete rows + // this scan never verified. protectedScanRoots := append(append([]string(nil), unreachableRoots...), suspectRoots...) + for _, unreadable := range unreadablePaths { + protectedScanRoots = appendUniquePath(protectedScanRoots, unreadable) + } for _, pending := range pendingEmptyScopes { beforeErrors := pending.result.Errors // forceDeleteAll only ever fires for confirmed cleanup, and even then @@ -1207,11 +1353,13 @@ func (s *Scanner) scanFolderByRoots( return nil, fmt.Errorf("syncing present library state for folder %d: %w", folder.ID, err) } - protectedRoots := unreachableRoots - if len(suspectRoots) > 0 { - protectedRoots = make([]string, 0, len(unreachableRoots)+len(suspectRoots)) - protectedRoots = append(append(protectedRoots, unreachableRoots...), suspectRoots...) - } + // Reuse the same protected set the scoped cleanup used, so membership + // removal and the trash sweep below honour roots the mid-loop re-probe + // found offline. Rebuilding from only the initial probe here would let a + // child that dropped during this scan have its already-missing rows hard + // deleted once they pass the removal grace — by the very scan that + // noticed the outage. + protectedRoots := protectedScanRoots removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots) if err != nil { return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) @@ -1349,6 +1497,9 @@ func removePath(paths []string, path string) []string { // deferred until the operator confirms or the files return. A root that // still has directory entries keeps the historical purge path. func (s *Scanner) suspectEmptyRoots(ctx context.Context, folderID int, configuredRoots, unreachableRoots []string) ([]string, error) { + if s == nil || s.fileRepo == nil { + return nil, nil + } unreachableSet := make(map[string]bool, len(unreachableRoots)) for _, root := range unreachableRoots { unreachableSet[root] = true @@ -1366,7 +1517,13 @@ func (s *Scanner) suspectEmptyRoots(ctx context.Context, folderID int, configure if len(emptyRoots) == 0 { return nil, nil } - suspect, err := s.fileRepo.ListRootsWithOnlyMissingFiles(ctx, folderID, emptyRoots) + // Any cataloged row under an empty-but-reachable root makes it suspect, + // not just a root whose rows are already all missing. Requiring the + // latter made this protection reactive: on the first scan after a mount + // dropped, the rows are still live, the root is not classified suspect, + // and the scan marks everything missing — the exact outage this guards + // against, recognised only in time to protect the wreckage. + suspect, err := s.fileRepo.ListRootsWithCatalogedFiles(ctx, folderID, emptyRoots) if err != nil { return nil, fmt.Errorf("listing suspect-empty roots for folder %d: %w", folderID, err) } @@ -1449,7 +1606,7 @@ func (s *Scanner) scanScope( return nil, fmt.Errorf("loading item statuses for folder %d path %q: %w", folder.ID, reconcileRoots[0], err) } - filePaths, walkErr := collectLogicalFilePaths(ctx, walkRoots, folder.Type) + filePaths, walkFailures, walkErr := collectLogicalFilePaths(ctx, walkRoots, folder.Type) if walkErr != nil { return nil, fmt.Errorf("walking media roots for %q: %w", reconcileRoots[0], walkErr) } @@ -1458,6 +1615,9 @@ func (s *Scanner) scanScope( "scope", firstScope(reconcileRoots), "files", len(filePaths), ) + if len(walkFailures) > 0 { + logIncompleteWalk(ctx, folder.ID, reconcileRoots, walkFailures) + } reportProgress(ctx, ProgressUpdate{ Phase: "processing", Message: "Processing discovered files", @@ -1609,6 +1769,7 @@ func (s *Scanner) scanScope( rootInference: rootInference, groupInference: groupInference, result: result, + walkFailures: walkFailures, }, nil } @@ -1628,35 +1789,49 @@ func (s *Scanner) applyScopedScan( return nil } + // Snapshot and group reconciliation prune whatever this walk did not see. + // That is only sound against a complete inventory: after a partial walk it + // would delete root snapshots, observed locations and group locations for + // the portion that could not be read, corrupting later metadata matching + // even though the media_files rows themselves stay protected below. + // Upserting what we did see is always safe; pruning is what must wait for + // a scan that read the whole tree. + // Pruning deletes whatever this walk did not observe, so it is only sound + // when the walk actually observed the scope. Three cases must suppress it: + // an incomplete walk (some of the tree was unreadable), a scope that was + // never walked at all (an unreachable root gets nil walkRoots), and a + // scope containing a protected path (a suspect-empty child compacted into + // a populated parent). In each case the observed set is a lower bound, and + // pruning against it deletes snapshots, observed locations and group + // locations for media that is still there — corrupting later matching even + // though the media_files rows themselves are protected. + // + // Keeping stale metadata is harmless by comparison: the next complete scan + // prunes it. + pruneUnseen := len(scope.walkFailures) == 0 && + len(scope.walkRoots) > 0 && + !anyPathWithinRoots(protectedRoots, scope.reconcileRoots) if err := s.reconcileScannedRoots( ctx, folder.ID, scope.reconcileRoots, scope.rootInference.Snapshots, + pruneUnseen, ); err != nil { return fmt.Errorf("reconciling scanned roots for scope %q: %w", scope.reconcileRoots[0], err) } - if err := s.reconcileScannedGroups(ctx, folder.ID, false, scope.reconcileRoots, true, scope.groupInference); err != nil { + if err := s.reconcileScannedGroups(ctx, folder.ID, false, scope.reconcileRoots, pruneUnseen, scope.groupInference); err != nil { return fmt.Errorf("reconciling scanned groups for scope %q: %w", scope.reconcileRoots[0], err) } - now := time.Now().UTC() - for _, existing := range scope.existingFiles { - if scope.seenPaths[existing.FilePath] { - continue - } - if existing.MissingSince == nil { - if err := s.fileRepo.MarkMissing(ctx, existing.ID, now); err != nil { - slog.ErrorContext(ctx, "scanner: failed to mark file missing", "component", "scanner", - "path", existing.FilePath, - "error", err, - ) - scope.result.Errors++ - continue - } - } - scope.result.Missing++ + // Wherever this scope's walk could not read, its picture of the disk is a + // lower bound. Protect exactly those paths from missing-marking rather + // than the whole scope, so one unreadable entry cannot permanently exempt + // a healthy root. + if len(scope.walkFailures) > 0 { + protectedRoots = append(append([]string(nil), protectedRoots...), scope.walkFailures...) } + s.markMissingExcludingProtected(ctx, folder.ID, scope.existingFiles, scope.seenPaths, protectedRoots, scope.result) staleFileIDs := collectStaleRemovedPathFileIDs(scope.existingFiles, scope.seenPaths, scope.reconcileRoots) if forceDeleteAll && len(scope.filePaths) == 0 { @@ -1685,10 +1860,191 @@ func mergeScanResult(dst *ScanResult, src *ScanResult) { dst.Updated += src.Updated dst.Unchanged += src.Unchanged dst.Errors += src.Errors + dst.MissingSkippedProtected += src.MissingSkippedProtected dst.RootObservations = append(dst.RootObservations, src.RootObservations...) dst.EmptyRootGuarded = dst.EmptyRootGuarded || src.EmptyRootGuarded } +// markMissingExcludingProtected marks every cataloged file the walk did not +// see as missing, except those under a protected root. +// +// Protection is the whole point: catalog reads filter on +// missing_since IS NULL, so marking a file is user-visible deletion. A root +// that is unreachable, is a suspect-empty mountpoint, or whose walk came back +// incomplete tells us nothing about whether its files exist, and hiding them +// on that basis turns a storage blip into a library outage. +// +// Shared by the folder scan and the scoped scan so the two cannot drift. +func (s *Scanner) markMissingExcludingProtected( + ctx context.Context, + folderID int, + existingFiles []*scanStateFile, + seenPaths map[string]bool, + protectedRoots []string, + result *ScanResult, +) { + now := time.Now().UTC() + for _, existing := range existingFiles { + if seenPaths[existing.FilePath] { + continue + } + if pathWithinAnyRoot(existing.FilePath, protectedRoots) { + result.MissingSkippedProtected++ + continue + } + // Only mark as missing if not already marked. + if existing.MissingSince == nil { + if err := s.fileRepo.MarkMissing(ctx, existing.ID, now); err != nil { + slog.ErrorContext(ctx, "scanner: failed to mark file missing", "component", "scanner", + "path", existing.FilePath, + "error", err, + ) + result.Errors++ + continue + } + } + result.Missing++ + } + logProtectedMissingSkips(ctx, folderID, result.MissingSkippedProtected, protectedRoots) +} + +// reprobeNestedRoots re-checks the configured roots nested strictly beneath +// parent and returns those that must now be protected, split by why. +// +// Root compaction folds a child mount into its parent for traversal, so a +// child that dies after the initial probe leaves no scope of its own and is +// never revisited by the post-walk re-probe, which only inspects scopes that +// walked empty. Without this, a child dropping mid-scan is indistinguishable +// from its contents having been deleted. +// +// Classification comes from ONE probe batch. Probing twice — once for +// reachability, then again for emptiness — leaves a window where a child that +// drops between the two samples is seen as reachable by the first and +// discarded by the second (which only returns reachable-and-empty roots), +// yielding no protection at all during exactly the disconnect this is meant +// to catch. +// +// cleanupArmed mirrors the caller's rule: an unreachable child is protected +// regardless, while a suspect-empty one yields to an explicit operator +// confirmation. +func (s *Scanner) reprobeNestedRoots( + ctx context.Context, + folderID int, + configuredRoots []string, + parent string, + cleanupArmed bool, +) (unreachable []string, suspect []string, err error) { + nested := make([]string, 0) + for _, root := range configuredRoots { + if root == parent { + continue + } + if pathWithinAnyRoot(root, []string{parent}) { + nested = append(nested, root) + } + } + if len(nested) == 0 { + return nil, nil, nil + } + + probes := rootcheck.ProbeManyWithTimeout(ctx, nested, rootcheck.DefaultProbeTimeout) + unreachable = make([]string, 0) + emptyRoots := make([]string, 0) + for i, root := range nested { + probe := probes[i] + switch { + case !probe.Reachable: + logUnreachableRoot(ctx, folderID, root, probe) + unreachable = append(unreachable, root) + case probe.Empty: + emptyRoots = append(emptyRoots, root) + } + } + + if cleanupArmed || len(emptyRoots) == 0 || s == nil || s.fileRepo == nil { + return unreachable, nil, nil + } + // An empty child that still owns cataloged rows is a lost mount, not an + // emptied library — the same rule suspectEmptyRoots applies, reusing the + // probe results already gathered above. + suspect, err = s.fileRepo.ListRootsWithCatalogedFiles(ctx, folderID, emptyRoots) + if err != nil { + return nil, nil, fmt.Errorf("listing suspect-empty nested roots for folder %d: %w", folderID, err) + } + if len(suspect) > 0 { + slog.WarnContext(ctx, "scanner: nested empty roots still hold cataloged files; protecting them from cleanup", + "component", "scanner", "folder_id", folderID, "roots", suspect) + } + return unreachable, suspect, nil +} + +// emptyCleanupArmed reports whether the operator has armed the folder's +// one-time empty-root cleanup allowance, WITHOUT consuming it. +// +// Consumption stays where it was, gated on what the scan actually found. This +// is only a read, so a scan that never reaches a consume path leaves the +// allowance armed for the next one. +func (s *Scanner) emptyCleanupArmed(ctx context.Context, folderID int) (bool, error) { + if s == nil || s.folderRepo == nil || folderID <= 0 { + return false, nil + } + folder, err := s.folderRepo.GetByID(ctx, folderID) + if err != nil { + return false, fmt.Errorf("reading cleanup allowance for folder %d: %w", folderID, err) + } + if folder == nil { + return false, nil + } + return folder.AllowEmptyCleanupOnce, nil +} + +// anyPathWithinRoots reports whether any of paths lies at or under one of +// roots. Used to detect a protected path inside a scope about to be pruned. +func anyPathWithinRoots(paths, roots []string) bool { + for _, path := range paths { + if pathWithinAnyRoot(path, roots) { + return true + } + } + return false +} + +// logIncompleteWalk reports that a scope's traversal could not read part of +// its tree, so its file list is a lower bound rather than an inventory. +func logIncompleteWalk(ctx context.Context, folderID int, reconcileRoots []string, walkFailures []string) { + slog.WarnContext(ctx, "scanner: walk could not read part of this scope; affected paths excluded from missing-file reconciliation", + "component", "scanner", + "folder_id", folderID, + "scope", firstScope(reconcileRoots), + "walk_failures", len(walkFailures), + "unreadable_paths", truncatePaths(walkFailures, 10), + ) +} + +// truncatePaths bounds a path list for logging; an outage can produce +// thousands and the first few identify the affected subtree well enough. +func truncatePaths(paths []string, limit int) []string { + if len(paths) <= limit { + return paths + } + return paths[:limit] +} + +// logProtectedMissingSkips reports files a scan declined to mark missing +// because their root was offline. This is the signal an operator needs to tell +// "my library shrank" from "my mount dropped": without it the scan looks +// clean while silently covering for absent storage. +func logProtectedMissingSkips(ctx context.Context, folderID, skipped int, protectedRoots []string) { + if skipped == 0 { + return + } + slog.WarnContext(ctx, "scanner: left files untouched under offline roots", "component", "scanner", + "folder_id", folderID, + "files_skipped", skipped, + "protected_roots", protectedRoots, + ) +} + func firstScope(scopes []string) string { if len(scopes) == 0 { return "" @@ -1701,6 +2057,7 @@ func mergeCleanupResult(dst *ScanResult, src *ScanResult, priorErrors int) { return } dst.Missing += src.Missing + dst.MissingSkippedProtected += src.MissingSkippedProtected dst.FilesDeleted += src.FilesDeleted if src.Errors > priorErrors { dst.Errors += src.Errors - priorErrors @@ -2108,6 +2465,7 @@ func (s *Scanner) ScanFile(ctx context.Context, filePath string, folder *models. folder.ID, nil, rootInference.Snapshots, + true, ); err != nil { return fmt.Errorf("reconciling scanned root for file: %w", err) } @@ -2993,11 +3351,17 @@ func filterGroupLocationsByScope(locations []models.MediaGroupLocation, scopePat return filtered } +// reconcileScannedRoots upserts the root snapshots this scan observed and, +// when pruneUnseen is set, deletes the snapshots in scope that it did not. +// Callers must pass pruneUnseen=false when the walk could not read part of the +// tree: the observed set is then a lower bound, and pruning against it drops +// snapshots for media that is still there. func (s *Scanner) reconcileScannedRoots( ctx context.Context, folderID int, scopeRoots []string, roots []models.ScannedMediaRoot, + pruneUnseen bool, ) error { if s == nil || s.rootSnapshotRepo == nil { return nil @@ -3019,6 +3383,9 @@ func (s *Scanner) reconcileScannedRoots( return err } + if !pruneUnseen { + return nil + } for scope, seenRoots := range seenByScope { if err := s.rootSnapshotRepo.DeleteMissingInScope(ctx, folderID, scope, seenRoots); err != nil { return err diff --git a/internal/scanner/scanner_test.go b/internal/scanner/scanner_test.go index 745ae29f..7786bc85 100644 --- a/internal/scanner/scanner_test.go +++ b/internal/scanner/scanner_test.go @@ -46,10 +46,13 @@ func TestCollectLogicalFilePaths_PreservesLogicalSymlinkRootPaths(t *testing.T) t.Skipf("symlinks not supported on this platform: %v", err) } - files, err := collectLogicalFilePaths(context.Background(), []string{logicalRoot}, "series") + files, walkFailures, err := collectLogicalFilePaths(context.Background(), []string{logicalRoot}, "series") if err != nil { t.Fatalf("collect logical paths: %v", err) } + if len(walkFailures) != 0 { + t.Fatalf("walkFailures = %v, want none for a fully readable tree", walkFailures) + } want := filepath.Join(logicalRoot, "Season 1", "Episode 01.mkv") if len(files) != 1 { @@ -82,10 +85,13 @@ func TestCollectLogicalFilePaths_DedupesSharedPhysicalDirsAndCycles(t *testing.T t.Skipf("symlinks not supported on this platform: %v", err) } - files, err := collectLogicalFilePaths(context.Background(), []string{physicalRoot, aliasRoot}, "movie") + files, walkFailures, err := collectLogicalFilePaths(context.Background(), []string{physicalRoot, aliasRoot}, "movie") if err != nil { t.Fatalf("collect logical paths: %v", err) } + if len(walkFailures) != 0 { + t.Fatalf("walkFailures = %v, want none for a fully readable tree", walkFailures) + } sort.Strings(files) want := []string{filepath.Join(physicalRoot, "Movie.mkv")} diff --git a/internal/scanner/types.go b/internal/scanner/types.go index f969c0aa..f8c531c3 100644 --- a/internal/scanner/types.go +++ b/internal/scanner/types.go @@ -11,16 +11,22 @@ type ScanResult struct { ItemsDeleted int Errors int EmptyRootGuarded bool + // MissingSkippedProtected counts files the scan did not find on disk but + // left alone because they sit under an unreachable or suspect-empty root. + // A non-zero value means the library is partly offline, not partly gone. + MissingSkippedProtected int // UnreachableRoots lists configured library roots that failed the - // reachability probe at scan start. Files under them were marked missing - // (hidden) but were exempted from trash emptying and item purging. + // reachability probe at scan start. Files under them are left untouched: + // not marked missing, not trashed, and not purged. A root that does not + // answer tells us nothing about whether its files still exist, and + // marking them missing would hide working titles from every catalog read + // until the next successful scan. UnreachableRoots []string // SuspectEmptyRoots lists roots that probed reachable but were literally // empty directories while the library still holds cataloged files under // them — the signature of a lost mount exposing its bare mountpoint - // directory. They receive the same cleanup exemptions as - // UnreachableRoots until the operator confirms cleanup or the files - // return. + // directory. They receive the same exemptions as UnreachableRoots until + // the operator confirms cleanup or the files return. SuspectEmptyRoots []string RootObservations []RootObservation } diff --git a/internal/scanqueue/service.go b/internal/scanqueue/service.go index c0ec05cb..2f9d9a5d 100644 --- a/internal/scanqueue/service.go +++ b/internal/scanqueue/service.go @@ -569,6 +569,7 @@ func scanResultFromIngest(result *libraryingest.Result) *evt.ScanRunResult { resp.Updated = result.ScanResult.Updated resp.Unchanged = result.ScanResult.Unchanged resp.Missing = result.ScanResult.Missing + resp.MissingSkippedProtected = result.ScanResult.MissingSkippedProtected resp.FilesDeleted = result.ScanResult.FilesDeleted resp.MembershipsRemoved = result.ScanResult.MembershipsRemoved resp.ItemsDeleted = result.ScanResult.ItemsDeleted