Compare commits

...
Author SHA1 Message Date
rxwatcher cd6b0afb34 feat: expose ebook library metadata setup 2026-06-08 13:33:43 +02:00
rxwatcher f734d7a01c feat: wire ebook metadata sync task 2026-06-08 13:33:43 +02:00
rxwatcher ddd579767b fix: harden ebook poster cache 2026-06-08 13:33:43 +02:00
rxwatcher 3018622e5e feat: add ebook metadata enricher 2026-06-08 13:33:43 +02:00
rxwatcher 9e7189f074 docs: clarify ebook foundation PR scope 2026-06-08 13:31:28 +02:00
rxwatcher 21170cfcf3 fix: address ebook scanner PR review 2026-06-08 12:29:35 +02:00
rxwatcher 0e1c2995a3 test: cover ebook series persistence decisions 2026-06-08 12:03:16 +02:00
rxwatcher 98d213204a feat: persist ebook series membership 2026-06-08 11:56:29 +02:00
rxwatcher 5ea8868047 fix: refresh ebook scan metadata safely 2026-06-08 11:52:20 +02:00
rxwatcher 0b97d2bd6d fix: preserve ebook scan people credits 2026-06-08 11:47:18 +02:00
rxwatcher 293db1f255 feat: scan ebook libraries in core 2026-06-08 11:40:31 +02:00
rxwatcher 1119cef018 fix: guard ebook subtree scans 2026-06-08 11:34:14 +02:00
rxwatcher c91a1d5b29 fix: handle ebook isbn labels 2026-06-08 11:31:45 +02:00
rxwatcher f15423d505 fix: harden ebook scanner foundation 2026-06-08 11:31:45 +02:00
rxwatcher 6de2b19c07 feat: add ebook scanner parser foundation 2026-06-08 11:31:45 +02:00
rxwatcher adf595cbf9 docs: plan ebook audiobook-parity implementation 2026-06-08 11:31:45 +02:00
rxwatcher 0b44952816 docs: define ebook architecture matching audiobooks 2026-06-08 11:31:45 +02:00
17 changed files with 3436 additions and 14 deletions
+16
View File
@@ -47,6 +47,7 @@ import (
"github.com/Silo-Server/silo-server/internal/clientip"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/database"
"github.com/Silo-Server/silo-server/internal/ebooks"
evt "github.com/Silo-Server/silo-server/internal/events"
"github.com/Silo-Server/silo-server/internal/historyimport"
"github.com/Silo-Server/silo-server/internal/imagecache"
@@ -888,6 +889,7 @@ func main() {
var seasonRepo *catalog.SeasonRepository
var episodeRepo *catalog.EpisodeRepository
var audiobookEnricher *audiobooks.Enricher
var ebookEnricher *ebooks.Enricher
if needsWorkers && deps.DB != nil && deps.FileRepo != nil {
chainRepo := metadata.NewChainRepository(deps.DB)
skippedRootRepo = metadata.NewSkippedRootRepository(deps.DB)
@@ -969,6 +971,14 @@ func main() {
personRepo,
providerIDRepo,
)
ebookEnricher = ebooks.NewEnricher(
deps.DB,
chainRepo,
pluginResolver,
itemRepo,
personRepo,
providerIDRepo,
)
// Always wire the image resolver so plugin-prefixed URLs (e.g.
// metadb://) can be resolved to presigned HTTP URLs in API responses.
@@ -991,6 +1001,9 @@ func main() {
audiobookEnricher.SetImageCacher(imageCacher)
audiobookEnricher.SetFFmpegPath(scanner.FFmpegPathFromFFprobe(scanner.FFprobePathFromFFmpeg(cfg.Playback.FFmpegPath)))
}
if ebookEnricher != nil {
ebookEnricher.SetImageCacher(imageCacher)
}
}
matchWorker = metadata.NewMatchWorker(metadataService, deps.FileRepo, cfg.Matcher.Workers, cfg.Matcher.BatchSize, 30*time.Second)
@@ -1534,6 +1547,9 @@ func main() {
if audiobookEnricher != nil {
taskMgr.Register(tasks.NewSyncAudiobookMetadataTask(audiobookEnricher))
}
if ebookEnricher != nil {
taskMgr.Register(tasks.NewSyncEbookMetadataTask(ebookEnricher))
}
if pluginInstallationStore != nil && pluginRuntimeConfigStore != nil && pluginService != nil {
pluginTasks, err := plugins.NewTaskRegistryWithTypedResolver(pluginInstallationStore, pluginRuntimeConfigStore, pluginService).Tasks(appCtx)
if err != nil {
@@ -0,0 +1,466 @@
# Ebooks Like Audiobooks Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Add ebooks as a core silo media type by mirroring the audiobook scanner architecture in `main`. Metadata enrichment is documented below as follow-up PR work, not part of the foundation PR.
**Architecture:** The foundation PR scans ebooks through core `internal/scanner` code. Later PRs add a core `internal/ebooks.Enricher` task that resolves plugin metadata providers with `content_level = 'ebook'`. The first-party `silo-plugin-ebook-metadata` provides lookup data; there is no ebook scanner plugin and no ebook details repository/table.
**Tech Stack:** Go, PostgreSQL Goose migrations, Silo scanner/catalog repositories, Silo metadata plugin chain, taskmanager scheduled tasks.
---
## File Structure
- `internal/scanner/ebook.go`: local ebook parser and format support.
- `internal/scanner/ebook_scan.go`: ebook scan flow matching `audiobook_scan.go`.
- `internal/scanner/scanner.go`: route `ebook(s)` libraries to the ebook scanner and add ebook walk mode.
- `migrations/sql/20260608000100_ebook_series.sql`: `ebook_series` table equivalent to `audiobook_series`.
- Follow-up PR: `internal/ebooks/enrichment.go`: ebook metadata sweep equivalent to `internal/audiobooks/enrichment.go`.
- Follow-up PR: `internal/taskmanager/tasks/sync_ebook_metadata.go`: scheduled/manual metadata task equivalent to `sync_audiobook_metadata.go`.
- Follow-up PR: `cmd/silo/main.go`: wire `ebooks.NewEnricher` next to `audiobooks.NewEnricher`.
- Tests live beside changed files and should pin “authors only, ISBN only, content_level ebook.”
### Task 1: Scanner Routing And Local Parser
**Files:**
- Modify: `internal/scanner/scanner.go`
- Modify: `internal/scanner/scanner_test.go`
- Create/modify: `internal/scanner/ebook.go`
- Create/modify: `internal/scanner/ebook_test.go`
- [ ] **Step 1: Write or keep failing scanner/parser tests**
Ensure these tests exist:
```go
func TestIsEbookLibraryType(t *testing.T) {
cases := []struct {
in string
want bool
}{
{"ebooks", true},
{"ebook", true},
{"Ebook", true},
{" EBOOKS ", true},
{"audiobooks", false},
{"movies", false},
{"", false},
}
for _, tc := range cases {
if got := isEbookLibraryType(tc.in); got != tc.want {
t.Errorf("isEbookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
}
}
}
func TestWalkModeForEbookLibraryTypes(t *testing.T) {
for _, libraryType := range []string{"ebook", "ebooks", " EBOOKS "} {
if got := walkModeFor(libraryType); got != walkModeEbook {
t.Fatalf("walkModeFor(%q) = %v, want walkModeEbook", libraryType, got)
}
}
}
func TestSupportsEbookFile(t *testing.T) {
cases := []struct {
path string
want bool
}{
{"book.epub", true},
{"book.PDF", true},
{"book.azw3", true},
{"book.mp3", false},
{"movie.mkv", false},
}
for _, tc := range cases {
if got := SupportsEbookFile(tc.path); got != tc.want {
t.Errorf("SupportsEbookFile(%q) = %v, want %v", tc.path, got, tc.want)
}
}
}
```
- [ ] **Step 2: Run scanner/parser red test**
Run:
```bash
go test ./internal/scanner -run 'Test(IsEbook|WalkModeForEbook|WalkModeEbook|SupportsEbook|ParseEbook)' -count=1
```
Expected before implementation: fail for missing ebook mode/parser functions. Expected after existing parser work is kept: pass.
- [ ] **Step 3: Implement minimal parser and walk mode**
Implement:
```go
var ebookExtensions = map[string]bool{
".epub": true,
".pdf": true,
".mobi": true,
".azw": true,
".azw3": true,
".fb2": true,
}
func SupportsEbookFile(filePath string) bool {
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
}
func isEbookLibraryType(libraryType string) bool {
switch strings.ToLower(strings.TrimSpace(libraryType)) {
case "ebook", "ebooks":
return true
default:
return false
}
}
```
Add `walkModeEbook` and make `walkMode.acceptsExt` call `SupportsEbookFile`.
- [ ] **Step 4: Run scanner/parser green test**
Run the same command as Step 2. Expected: `ok github.com/Silo-Server/silo-server/internal/scanner`.
- [ ] **Step 5: Commit Task 1**
```bash
git add internal/scanner/scanner.go internal/scanner/scanner_test.go internal/scanner/ebook.go internal/scanner/ebook_test.go
git commit -m "feat: add ebook scanner parser foundation"
```
### Task 2: Ebook Core Scan Persistence
**Files:**
- Create: `internal/scanner/ebook_scan.go`
- Modify: `internal/scanner/ebook_test.go`
- Modify: `internal/scanner/scanner.go`
- [ ] **Step 1: Write failing tests for audiobook-shaped ebook helpers**
Add tests for:
```go
func TestEbookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
book := &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
if got := ebookIdentityConfidence(book); got != "high" {
t.Fatalf("complete metadata confidence = %q, want high", got)
}
book = &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024}
if got := ebookIdentityConfidence(book); got != "medium" {
t.Fatalf("partial metadata confidence = %q, want medium", got)
}
book = &parsedEbook{}
if got := ebookIdentityConfidence(book); got != "low" {
t.Fatalf("empty metadata confidence = %q, want low", got)
}
}
func TestResolveEbookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
finder := &fakeRootContentFinder{}
writer := &fakeFilesystemItemWriter{}
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{
Title: "Book", Year: 2024, Authors: []string{"Author"}, Description: "Overview", Publisher: "Publisher", Genres: []string{"Fiction"}, Language: "en",
})
if err != nil {
t.Fatalf("resolveEbookMediaItem: %v", err)
}
if got == "" || len(writer.upserts) != 1 {
t.Fatalf("contentID/upserts = %q/%d, want id and one upsert", got, len(writer.upserts))
}
item := writer.upserts[0]
if item.Type != "ebook" || item.Title != "Book" || item.Year != 2024 || item.Overview != "Overview" || item.OriginalLanguage != "en" {
t.Fatalf("upserted ebook item = %+v", item)
}
}
```
- [ ] **Step 2: Run red helper tests**
Run:
```bash
go test ./internal/scanner -run 'Test(EbookIdentity|ResolveEbook|EbookPeople)' -count=1
```
Expected before implementation: fail with undefined ebook scan helper functions.
- [ ] **Step 3: Implement ebook scan helpers mirroring audiobook names**
Create `internal/scanner/ebook_scan.go` with functions shaped after `audiobook_scan.go`:
```go
func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolder) error
func (s *Scanner) reconcileEbookFile(ctx context.Context, folder *models.MediaFolder, filePath string, skipped *int64) error
func (s *Scanner) ebookFileShouldSkip(ctx context.Context, folder *models.MediaFolder, filePath string, size int64, modifiedAt time.Time) (bool, error)
func (s *Scanner) upsertEbookMediaItem(ctx context.Context, folderID int, filePath string, book *parsedEbook) (string, error)
func resolveEbookMediaItem(ctx context.Context, rootFinder filesystemRootContentFinder, itemWriter filesystemMediaItemWriter, folderID int, filePath string, book *parsedEbook) (string, error)
func createEbookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedEbook, cleanTitle string) (string, error)
func applyEbookToMediaItem(item *models.MediaItem, book *parsedEbook)
func (s *Scanner) upsertEbookMediaFile(ctx context.Context, folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) error
func (s *Scanner) upsertEbookPeople(ctx context.Context, contentID string, book *parsedEbook) error
func (s *Scanner) upsertEbookSeries(ctx context.Context, contentID string, book *parsedEbook) error
func ebookIdentityConfidence(book *parsedEbook) string
```
Behavior:
- `ScanEbookFolder` walks `folder.Paths`, collects files where `SupportsEbookFile` is true, processes in a worker pool like `ScanAudiobookFolder`, logs `ebook scan: starting/progress/completed/indexed`.
- `upsertEbookMediaItem` reuses `FindContentIDByRootPath(ctx, folderID, filePath, "ebook")`, then finds by existing `media_files.file_path`, then creates a new `media_items` row.
- `applyEbookToMediaItem` sets type/title/year/overview/publisher-as-studio/genres/release date/original language.
- `upsertEbookMediaFile` writes one `media_files` row with `CanonicalRootPath`, `ObservedRootPath`, `ContentGroupKey`, `FilePath` all tied to the ebook file path or content ID in the same style as audiobook file persistence.
- `upsertEbookPeople` writes author credits only with `models.PersonKindAuthor`; it never writes `models.PersonKindNarrator`.
- `reconcileEbookFile` writes ISBN provider ID with:
```sql
INSERT INTO media_item_provider_ids (content_id, provider, provider_id, item_type)
VALUES ($1, 'isbn', $2, 'ebook')
ON CONFLICT DO NOTHING
```
- [ ] **Step 4: Route ebook libraries in `ScanFolder`**
Add this block beside audiobook/podcast routing:
```go
if isEbookLibraryType(folder.Type) {
if err := s.ScanEbookFolder(watchCtx, folder); err != nil {
return nil, err
}
return &ScanResult{}, nil
}
```
- [ ] **Step 5: Run green helper tests**
Run:
```bash
go test ./internal/scanner -run 'Test(EbookIdentity|ResolveEbook|EbookPeople|ScanEbook)' -count=1
```
Expected: scanner tests pass.
- [ ] **Step 6: Commit Task 2**
```bash
git add internal/scanner/scanner.go internal/scanner/ebook_scan.go internal/scanner/ebook_test.go
git commit -m "feat: scan ebook libraries in core"
```
### Task 3: Ebook Series Migration
**Files:**
- Create: `migrations/sql/20260608000100_ebook_series.sql`
- Modify: `internal/scanner/ebook_scan.go`
- [ ] **Step 1: Add migration mirroring `audiobook_series`**
Create:
```sql
-- +goose Up
-- +goose StatementBegin
CREATE TABLE ebook_series (
content_id TEXT PRIMARY KEY REFERENCES media_items(content_id) ON DELETE CASCADE,
series_name TEXT NOT NULL,
series_index NUMERIC,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX ebook_series_name_lower
ON ebook_series (LOWER(series_name), series_index NULLS LAST);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS ebook_series;
-- +goose StatementEnd
```
- [ ] **Step 2: Add scanner series upsert SQL**
Use the same write/delete/skip-identical shape as `upsertAudiobookSeries`, with table name `ebook_series` and `book.SeriesIndex` parsed through existing `parseSeriesIndex`.
- [ ] **Step 3: Run migration and scanner tests**
Run:
```bash
go test ./internal/scanner -run 'Test(Ebook|ResolveEbook)' -count=1
```
Expected: pass.
- [ ] **Step 4: Commit Task 3**
```bash
git add migrations/sql/20260608000100_ebook_series.sql internal/scanner/ebook_scan.go
git commit -m "feat: persist ebook series membership"
```
### Follow-up PR Task 4: Ebook Metadata Enricher
**Files:**
- Create: `internal/ebooks/enrichment.go`
- Create: `internal/ebooks/enrichment_test.go`
- [ ] **Step 1: Copy audiobook enricher structure with ebook domain changes**
Create `internal/ebooks/enrichment.go` from `internal/audiobooks/enrichment.go` and change:
- package `ebooks`
- log prefix `ebook enrichment`
- default env var `SILO_EBOOK_ENRICH_WORKERS`
- batch query `WHERE mi.type = 'ebook'`
- chain resolution `metadata.ResolveChain(ctx, item.FolderID, "ebook", e.chainRepo, e.resolver)`
- search query `ContentType: "ebook"`
- metadata request `ContentType: "ebook"`
- people persistence filters to author credits only; discard narrator credits if a plugin returns them
- no local ffmpeg audiobook cover fallback
- [ ] **Step 2: Add tests for chain content level and authors-only people**
Add tests that instantiate an `Enricher` with fake provider behavior where possible, or unit-test narrow helpers:
```go
func TestEbookEnricherContentType(t *testing.T) {
if got := ebookContentType(); got != "ebook" {
t.Fatalf("ebookContentType() = %q, want ebook", got)
}
}
func TestFilterEbookPeopleKeepsAuthorsOnly(t *testing.T) {
in := []models.ItemPerson{
{Person: models.Person{Name: "Author"}, Kind: models.PersonKindAuthor},
{Person: models.Person{Name: "Narrator"}, Kind: models.PersonKindNarrator},
}
got := filterEbookPeople(in)
if len(got) != 1 || got[0].Kind != models.PersonKindAuthor || got[0].Person.Name != "Author" {
t.Fatalf("filterEbookPeople() = %+v, want author only", got)
}
}
```
- [ ] **Step 3: Run red/green ebook enricher tests**
Run:
```bash
go test ./internal/ebooks -count=1
```
Expected after implementation: pass.
- [ ] **Step 4: Commit Task 4**
```bash
git add internal/ebooks/enrichment.go internal/ebooks/enrichment_test.go
git commit -m "feat: add ebook metadata enricher"
```
### Follow-up PR Task 5: Ebook Metadata Task And Main Wiring
**Files:**
- Create: `internal/taskmanager/tasks/sync_ebook_metadata.go`
- Modify: `cmd/silo/main.go`
- [ ] **Step 1: Add sync task matching audiobook task**
Create task:
```go
type SyncEbookMetadataTask struct {
enricher *ebooks.Enricher
}
func NewSyncEbookMetadataTask(enricher *ebooks.Enricher) *SyncEbookMetadataTask {
return &SyncEbookMetadataTask{enricher: enricher}
}
func (t *SyncEbookMetadataTask) Key() string { return "sync_ebook_metadata" }
func (t *SyncEbookMetadataTask) Name() string { return "Sync Ebook Metadata" }
func (t *SyncEbookMetadataTask) Description() string {
return "Fetches metadata (cover art, overview, authors) for ebooks that have not yet been enriched"
}
func (t *SyncEbookMetadataTask) Category() taskmanager.TaskCategory {
return taskmanager.TaskCategoryMetadata
}
func (t *SyncEbookMetadataTask) IsHidden() bool { return false }
func (t *SyncEbookMetadataTask) DefaultTriggers() []taskmanager.TriggerConfig {
return []taskmanager.TriggerConfig{{Type: taskmanager.TriggerTypeInterval, IntervalMs: 5 * 60 * 1000}}
}
```
- [ ] **Step 2: Wire `ebooks.NewEnricher` beside audiobook enricher**
In `cmd/silo/main.go`:
- add import `github.com/Silo-Server/silo-server/internal/ebooks`
- add `var ebookEnricher *ebooks.Enricher`
- construct it with the same dependencies as `audiobooks.NewEnricher`
- set image cacher if object storage is available and the ebook enricher supports remote image caching
- register `tasks.NewSyncEbookMetadataTask(ebookEnricher)` beside `NewSyncAudiobookMetadataTask`
- [ ] **Step 3: Run task/main build tests**
Run:
```bash
go test ./internal/taskmanager/tasks ./internal/ebooks -count=1
go test ./cmd/silo -run TestNonExistent -count=1
```
Expected: packages compile and tests pass.
- [ ] **Step 4: Commit Task 5**
```bash
git add cmd/silo/main.go internal/taskmanager/tasks/sync_ebook_metadata.go
git commit -m "feat: wire ebook metadata sync task"
```
### Task 6: Verification And PR Cleanup
**Files:**
- Review all modified files.
- [ ] **Step 1: Run focused Go tests**
```bash
go test ./internal/scanner ./internal/ebooks ./internal/taskmanager/tasks -count=1
```
Expected: pass.
- [ ] **Step 2: Run broader compile/test set**
```bash
go test ./internal/catalog ./internal/metadata ./cmd/silo -count=1
```
Expected: pass, or record unrelated baseline failures with exact package/error.
- [ ] **Step 3: Check no wrong architecture remains**
Run:
```bash
rg -n "ebook_details|silo-plugin-ebooks|PersonKindNarrator|asin" internal/scanner internal/ebooks docs/superpowers/specs docs/superpowers/plans
```
Expected:
- no `ebook_details`
- no scanner dependency on `silo-plugin-ebooks`
- no ebook code writing `PersonKindNarrator`
- no ebook code writing ASIN
- [ ] **Step 4: Commit final cleanup if needed**
```bash
git status --short
git add <only ebook-related cleanup files>
git commit -m "test: verify ebook audiobook-parity path"
```
Only commit if Step 1-3 caused file changes.
@@ -0,0 +1,102 @@
# Ebooks Like Audiobooks Design
## Goal
Add ebooks to silo by matching the audiobook architecture in `main` as closely
as possible. Ebooks are a core media type in `silo-server`; only metadata
provider lookup lives in a plugin.
## Non-Goals
- Do not add or keep a scanner/runtime plugin named `silo-plugin-ebooks`.
- Do not treat ebooks as an audiobook sub-feature.
- Do not add ebook narrators.
- Do not add ebook ASIN handling.
- Do not introduce an ebook details repository/table unless the audiobook path
gains the same shape first.
## Audiobook Pattern In Main
Audiobooks have three cooperating pieces:
1. Core scan:
- `internal/scanner/scanner.go` routes `audiobook` and `audiobooks` library
types to `ScanAudiobookFolder`.
- `internal/scanner/audiobook_scan.go` writes `media_items`,
`media_files`, `item_people`, `audiobook_series`,
`media_item_libraries`, and durable provider IDs.
2. Core metadata enrichment:
- `cmd/silo/main.go` wires `audiobooks.NewEnricher`.
- `internal/taskmanager/tasks/sync_audiobook_metadata.go` registers the
periodic/manual task.
- `internal/audiobooks/enrichment.go` sweeps unenriched `audiobook` items,
resolves the configured metadata provider chain at
`content_level = 'audiobook'`, calls `Search` then `GetMetadata`, and
persists provider IDs, poster/overview/scalar metadata, and people.
3. First-party metadata plugin:
- `silo-plugin-audiobook-metadata` exposes `metadata_provider.v1` with an
audiobook capability and provider implementations.
## Ebook Equivalent Across PRs
Ebooks should mirror that shape:
1. Core scan:
- `internal/scanner/scanner.go` routes `ebook` and `ebooks` libraries to an
ebook scan path.
- `internal/scanner/ebook.go` parses local ebook metadata from supported
files.
- `internal/scanner/ebook_scan.go` writes:
- `media_items.type = 'ebook'`
- `media_files.base_type = 'ebook'`
- `item_people` author credits only
- `ebook_series`
- `media_item_libraries`
- `media_item_provider_ids` using provider `isbn` when an ISBN is parsed
2. Follow-up core metadata enrichment:
- Add `internal/ebooks/enrichment.go`, shaped after
`internal/audiobooks/enrichment.go`.
- Add `internal/taskmanager/tasks/sync_ebook_metadata.go`, shaped after
`sync_audiobook_metadata.go`.
- Wire `ebooks.NewEnricher` in `cmd/silo/main.go` next to the audiobook
enricher.
- Resolve provider chains at `content_level = 'ebook'`.
- Search and fetch metadata with `ContentType = "ebook"`.
- Persist provider IDs, poster/overview/scalar metadata, and author people.
3. First-party metadata plugin:
- Use `silo-plugin-ebook-metadata` as the first-party ebook metadata
provider plugin.
- The core app should rely on that plugin for provider-specific metadata
lookups, just as audiobooks rely on `silo-plugin-audiobook-metadata`.
## Ebook Domain Rules
- Ebooks have ISBNs.
- Ebooks have authors.
- Ebooks do not have narrators.
- Ebooks do not have ASINs.
- Ebook metadata plugins may return provider IDs, but core scanner-derived ISBN
should be stored under provider `isbn`.
- Local parsing may read embedded ebook metadata from EPUB/FB2 and format/file
facts from other supported ebook formats.
## First PR Scope
The first implementation PR should establish the mirrored foundation:
- ebook library type and walk mode
- local ebook parser
- ebook scan routing and persistence
- `ebook_series` migration
- tests proving ebook authors-only behavior, ISBN provider ID behavior, and
scanner-owned ebook series behavior
Deferred follow-up PR scope:
- ebook metadata enrichment package/task/wiring, analogous to
`internal/audiobooks.Enricher`
- plugin chain resolution at `content_level = 'ebook'`
UI/detail page, ebook reader experience, ABS ebook compatibility endpoints, and
advanced provider-specific plugin improvements can be separate PRs unless they
are required to prove the core path works.
+19 -12
View File
@@ -2029,18 +2029,8 @@ func (h *LibraryHandler) seedDefaultChain(ctx context.Context, libraryType strin
return nil
}
// Determine which content levels apply.
var levels []string
switch libraryType {
case "series":
levels = []string{"series", "season", "episode"}
case "movies", "movie":
levels = []string{"movie"}
case "audiobooks", "audiobook":
levels = []string{"audiobook"}
case "mixed":
levels = []string{"movie", "series", "season", "episode", "audiobook"}
default:
levels := metadataContentLevelsForLibraryType(libraryType)
if len(levels) == 0 {
return nil
}
@@ -2093,6 +2083,23 @@ func (h *LibraryHandler) seedDefaultChain(ctx context.Context, libraryType strin
return entries
}
func metadataContentLevelsForLibraryType(libraryType string) []string {
switch libraryType {
case "series":
return []string{"series", "season", "episode"}
case "movies", "movie":
return []string{"movie"}
case "audiobooks", "audiobook":
return []string{"audiobook"}
case "ebooks", "ebook":
return []string{"ebook"}
case "mixed":
return []string{"movie", "series", "season", "episode", "audiobook", "ebook"}
default:
return nil
}
}
// HandleListStaleIDs handles GET /libraries/stale-ids.
func (h *LibraryHandler) HandleListStaleIDs(w http.ResponseWriter, r *http.Request) {
if h.StaleIDRepo == nil {
@@ -0,0 +1,26 @@
package handlers
import (
"reflect"
"testing"
)
func TestMetadataContentLevelsForLibraryTypeIncludesEbooks(t *testing.T) {
cases := []struct {
name string
libraryType string
want []string
}{
{name: "plural ebooks", libraryType: "ebooks", want: []string{"ebook"}},
{name: "singular ebook", libraryType: "ebook", want: []string{"ebook"}},
{name: "mixed includes ebook", libraryType: "mixed", want: []string{"movie", "series", "season", "episode", "audiobook", "ebook"}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := metadataContentLevelsForLibraryType(tc.libraryType); !reflect.DeepEqual(got, tc.want) {
t.Fatalf("metadataContentLevelsForLibraryType(%q) = %#v, want %#v", tc.libraryType, got, tc.want)
}
})
}
}
+602
View File
@@ -0,0 +1,602 @@
package ebooks
// Enricher periodically enriches ebook media_items that are missing metadata
// by querying the configured metadata-provider chain for each item's library
// folder.
import (
"context"
"fmt"
"log/slog"
"os"
"reflect"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/metadata"
"github.com/Silo-Server/silo-server/internal/models"
)
const (
ebookMetadataImageProviderID = "ebook-metadata"
defaultEnrichBatchSize = 50
defaultEnrichWorkers = 4
)
func ebookContentType() string {
return "ebook"
}
func ebookEnrichWorkers() int {
n := defaultEnrichWorkers
if v := os.Getenv("SILO_EBOOK_ENRICH_WORKERS"); v != "" {
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
n = parsed
}
}
if n > defaultEnrichBatchSize {
n = defaultEnrichBatchSize
}
return n
}
type enrichmentItemRow struct {
ContentID string
Title string
Year int
FolderID int
Language string
Author string
ProviderIDs map[string]string
}
// Enricher drives the ebook metadata enrichment sweep.
type Enricher struct {
pool *pgxpool.Pool
chainRepo *metadata.ChainRepository
resolver *metadata.PluginResolverAdapter
itemRepo *catalog.ItemRepository
personRepo *catalog.PersonRepository
providerIDs *catalog.ProviderIDRepository
imageCacher metadata.ImageCacher
batchSize int
workers int
}
func NewEnricher(
pool *pgxpool.Pool,
chainRepo *metadata.ChainRepository,
resolver *metadata.PluginResolverAdapter,
itemRepo *catalog.ItemRepository,
personRepo *catalog.PersonRepository,
providerIDs *catalog.ProviderIDRepository,
) *Enricher {
return &Enricher{
pool: pool,
chainRepo: chainRepo,
resolver: resolver,
itemRepo: itemRepo,
personRepo: personRepo,
providerIDs: providerIDs,
batchSize: defaultEnrichBatchSize,
workers: ebookEnrichWorkers(),
}
}
func (e *Enricher) SetImageCacher(cacher metadata.ImageCacher) {
if e == nil {
return
}
e.imageCacher = cacher
}
func (e *Enricher) Run(ctx context.Context) (int, error) {
if e == nil || e.pool == nil || e.chainRepo == nil {
return 0, nil
}
items, err := e.claimBatch(ctx)
if err != nil {
return 0, fmt.Errorf("ebook enrichment: claim batch: %w", err)
}
if len(items) == 0 {
return 0, nil
}
slog.Info("ebook enrichment: sweep started",
"count", len(items),
"workers", e.workers,
)
enriched := e.runBatch(ctx, items, e.enrichItem)
slog.Info("ebook enrichment: sweep complete",
"attempted", len(items),
"enriched", enriched,
)
return enriched, nil
}
func (e *Enricher) runBatch(ctx context.Context, items []enrichmentItemRow, enrichFn func(context.Context, enrichmentItemRow) error) int {
workers := e.workers
if workers <= 0 {
workers = 1
}
if workers > len(items) {
workers = len(items)
}
ch := make(chan enrichmentItemRow, workers)
var (
wg sync.WaitGroup
enriched int64
)
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for item := range ch {
if ctx.Err() != nil {
continue
}
if err := enrichFn(ctx, item); err != nil {
slog.Warn("ebook enrichment: item failed",
"content_id", item.ContentID,
"title", item.Title,
"error", err,
)
continue
}
atomic.AddInt64(&enriched, 1)
}
}()
}
for _, item := range items {
if ctx.Err() != nil {
break
}
ch <- item
}
close(ch)
wg.Wait()
return int(enriched)
}
func (e *Enricher) claimBatch(ctx context.Context) ([]enrichmentItemRow, error) {
rows, err := e.pool.Query(ctx, `
SELECT
mi.content_id,
mi.title,
mi.year,
COALESCE(mil.media_folder_id, 0) AS folder_id,
COALESCE(mf.metadata_language, 'en') AS language,
COALESCE(
(SELECT p.name
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
AND ip.kind = 7
ORDER BY ip.sort_order, ip.id
LIMIT 1),
''
) AS author
FROM media_items mi
LEFT JOIN media_item_libraries mil ON mil.content_id = mi.content_id
LEFT JOIN media_folders mf ON mf.id = mil.media_folder_id
WHERE mi.type = 'ebook'
AND (mi.poster_path IS NULL OR mi.poster_path = '')
AND mi.last_refreshed IS NULL
ORDER BY mi.created_at ASC
LIMIT $1
`, e.batchSize)
if err != nil {
return nil, fmt.Errorf("querying unenriched ebooks: %w", err)
}
defer rows.Close()
var items []enrichmentItemRow
seen := make(map[string]struct{})
for rows.Next() {
var item enrichmentItemRow
if err := rows.Scan(
&item.ContentID,
&item.Title,
&item.Year,
&item.FolderID,
&item.Language,
&item.Author,
); err != nil {
return nil, fmt.Errorf("scanning ebook enrichment row: %w", err)
}
if _, dup := seen[item.ContentID]; dup {
continue
}
seen[item.ContentID] = struct{}{}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterating ebook enrichment rows: %w", err)
}
if e.providerIDs != nil {
for i := range items {
pids, err := e.providerIDs.GetByContentID(ctx, items[i].ContentID)
if err == nil {
items[i].ProviderIDs = providerIDMapFromRows(pids)
}
}
}
return items, nil
}
func (e *Enricher) enrichItem(ctx context.Context, item enrichmentItemRow) error {
if item.FolderID == 0 {
slog.Debug("ebook enrichment: item has no library folder, skipping",
"content_id", item.ContentID,
"title", item.Title,
)
return e.stampLastRefreshed(ctx, item.ContentID)
}
providers, err := metadata.ResolveChain(ctx, item.FolderID, ebookContentType(), e.chainRepo, e.resolver)
if err != nil {
return fmt.Errorf("resolving ebook chain for folder %d: %w", item.FolderID, err)
}
if len(providers) == 0 {
slog.Debug("ebook enrichment: no providers in chain",
"content_id", item.ContentID,
"folder_id", item.FolderID,
)
return e.stampLastRefreshed(ctx, item.ContentID)
}
accumulatedIDs := make(map[string]string, len(item.ProviderIDs))
for k, v := range item.ProviderIDs {
accumulatedIDs[k] = v
}
searchQuery := metadata.SearchQuery{
Title: item.Title,
Year: item.Year,
ContentType: ebookContentType(),
ProviderIDs: accumulatedIDs,
Language: item.Language,
}
for _, p := range providers {
sp, ok := p.(metadata.SearchProvider)
if !ok {
continue
}
results, searchErr := sp.Search(ctx, searchQuery)
if searchErr != nil {
slog.Warn("ebook enrichment: search error",
"provider", p.Slug(),
"content_id", item.ContentID,
"error", searchErr,
)
continue
}
if len(results) == 0 {
continue
}
for k, v := range results[0].ProviderIDs {
if v != "" {
if _, exists := accumulatedIDs[k]; !exists {
accumulatedIDs[k] = v
}
}
}
slog.Debug("ebook enrichment: search result",
"provider", p.Slug(),
"content_id", item.ContentID,
"matched_ids", accumulatedIDs,
)
}
accumulator := &metadata.MetadataResult{
ProviderIDs: accumulatedIDs,
}
for _, p := range providers {
mp, ok := p.(metadata.MetadataProvider)
if !ok {
continue
}
result, getErr := mp.GetMetadata(ctx, metadata.MetadataRequest{
ProviderIDs: accumulatedIDs,
ContentType: ebookContentType(),
Language: item.Language,
})
if getErr != nil {
slog.Warn("ebook enrichment: GetMetadata error",
"provider", p.Slug(),
"content_id", item.ContentID,
"error", getErr,
)
continue
}
if result == nil || !result.HasMetadata {
continue
}
mergeEnrichmentProviderIDs(accumulator, result)
accumulatedIDs = accumulator.ProviderIDs
metadata.MergeMetadata(result, accumulator, nil, metadata.MergeFillEmpty)
slog.Debug("ebook enrichment: metadata received",
"provider", p.Slug(),
"content_id", item.ContentID,
"has_poster", result.PosterPath != "",
"has_overview", result.Overview != "",
)
}
if !accumulator.HasMetadata && accumulator.PosterPath == "" && accumulator.Overview == "" {
slog.Info("ebook enrichment: no metadata found",
"content_id", item.ContentID,
"title", item.Title,
)
return e.stampLastRefreshed(ctx, item.ContentID)
}
e.cacheRemotePoster(ctx, item.ContentID, accumulator)
if err := e.persist(ctx, item.ContentID, accumulatedIDs, accumulator); err != nil {
return fmt.Errorf("persisting enrichment for %s: %w", item.ContentID, err)
}
slog.Info("ebook enrichment: enriched",
"content_id", item.ContentID,
"title", item.Title,
"poster", accumulator.PosterPath != "",
"overview", accumulator.Overview != "",
"people", len(filterEbookPeople(accumulator.People)),
)
return nil
}
func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, result *metadata.MetadataResult) {
if e == nil || result == nil || result.PosterPath == "" {
return
}
if !strings.HasPrefix(result.PosterPath, "http://") && !strings.HasPrefix(result.PosterPath, "https://") {
return
}
if isNilImageCacher(e.imageCacher) {
return
}
cached, err := e.imageCacher.CacheImage(ctx, metadata.CacheImageRequest{
SourceURL: result.PosterPath,
ProviderID: ebookMetadataImageProviderID,
ContentType: "ebooks",
ContentID: contentID,
ImageType: metadata.ImagePoster,
})
if err != nil {
slog.Warn("ebook enrichment: poster cache failed, keeping provider URL",
"content_id", contentID,
"url", result.PosterPath,
"error", err,
)
return
}
if cached == nil {
slog.Warn("ebook enrichment: poster cache returned no result, keeping provider URL",
"content_id", contentID,
"url", result.PosterPath,
)
return
}
if storedPath := cachedOriginalImagePath(cached.BasePath, cached.Ext); storedPath != "" {
result.PosterPath = storedPath
}
if cached.Thumbhash != "" {
result.PosterThumbhash = cached.Thumbhash
}
}
func isNilImageCacher(cacher metadata.ImageCacher) bool {
if cacher == nil {
return true
}
value := reflect.ValueOf(cacher)
switch value.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
return value.IsNil()
default:
return false
}
}
func cachedOriginalImagePath(basePath, ext string) string {
if basePath == "" {
return ""
}
if strings.Contains(basePath, "/original.") {
return basePath
}
if ext == "" {
ext = ".jpg"
}
return strings.TrimRight(basePath, "/") + "/original" + ext
}
func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs map[string]string, result *metadata.MetadataResult) error {
upd := &catalog.MetadataUpdate{}
if result.PosterPath != "" {
upd.PosterPath = &result.PosterPath
}
if result.PosterThumbhash != "" {
upd.PosterThumbhash = &result.PosterThumbhash
}
if result.BackdropPath != "" {
upd.BackdropPath = &result.BackdropPath
}
if result.BackdropThumbhash != "" {
upd.BackdropThumbhash = &result.BackdropThumbhash
}
if result.LogoPath != "" {
upd.LogoPath = &result.LogoPath
}
if result.Overview != "" {
upd.Overview = &result.Overview
}
if result.Tagline != "" {
upd.Tagline = &result.Tagline
}
if result.ReleaseDate != "" {
upd.ReleaseDate = &result.ReleaseDate
}
if len(result.Genres) > 0 {
genres := append([]string(nil), result.Genres...)
upd.Genres = &genres
}
if len(result.Studios) > 0 {
studios := append([]string(nil), result.Studios...)
upd.Studios = &studios
}
if result.ContentRating != "" {
upd.ContentRating = &result.ContentRating
}
if result.Runtime > 0 {
upd.Runtime = &result.Runtime
}
if result.Year > 0 {
upd.Year = &result.Year
}
if e.providerIDs != nil && len(providerIDs) > 0 {
if err := e.providerIDs.ReplaceByContentID(ctx, contentID, providerIDs); err != nil {
slog.Warn("ebook enrichment: failed to persist provider IDs",
"content_id", contentID,
"error", err,
)
}
}
if err := e.updateMetadataAndTimestamps(ctx, contentID, upd); err != nil {
return err
}
authors := filterEbookPeople(result.People)
if len(authors) > 0 && e.personRepo != nil && e.itemRepo != nil {
if err := e.persistPeople(ctx, contentID, authors); err != nil {
slog.Warn("ebook enrichment: failed to persist people",
"content_id", contentID,
"error", err,
)
}
}
return nil
}
func (e *Enricher) updateMetadataAndTimestamps(ctx context.Context, contentID string, upd *catalog.MetadataUpdate) error {
if e.itemRepo == nil {
return nil
}
if err := e.itemRepo.UpdateMetadata(ctx, contentID, upd); err != nil {
return fmt.Errorf("UpdateMetadata: %w", err)
}
return e.stampLastRefreshed(ctx, contentID)
}
func (e *Enricher) stampLastRefreshed(ctx context.Context, contentID string) error {
if e.pool == nil {
return nil
}
now := time.Now().UTC()
_, err := e.pool.Exec(ctx, `
UPDATE media_items
SET last_refreshed = $1,
matched_at = COALESCE(matched_at, $1),
status = CASE WHEN status = 'pending' THEN 'matched' ELSE status END
WHERE content_id = $2
`, now, contentID)
return err
}
func (e *Enricher) persistPeople(ctx context.Context, contentID string, people []models.ItemPerson) error {
people = filterEbookPeople(people)
if len(people) == 0 {
return nil
}
persons := make([]models.Person, len(people))
for i := range people {
persons[i] = people[i].Person
}
personIDs, err := e.personRepo.BatchFindOrCreate(ctx, persons)
if err != nil {
return fmt.Errorf("BatchFindOrCreate people: %w", err)
}
linked := make([]models.ItemPerson, 0, len(people))
for i := range people {
if i >= len(personIDs) || personIDs[i] == 0 {
continue
}
ip := people[i]
ip.Person.ID = personIDs[i]
linked = append(linked, ip)
}
if len(linked) == 0 {
return nil
}
return e.itemRepo.ReplacePeople(ctx, contentID, linked)
}
func filterEbookPeople(people []models.ItemPerson) []models.ItemPerson {
authors := make([]models.ItemPerson, 0, len(people))
for _, person := range people {
if person.Kind != models.PersonKindAuthor {
continue
}
person.SortOrder = len(authors)
authors = append(authors, person)
}
return authors
}
func mergeEnrichmentProviderIDs(dst *metadata.MetadataResult, src *metadata.MetadataResult) {
if src == nil || len(src.ProviderIDs) == 0 {
return
}
if dst.ProviderIDs == nil {
dst.ProviderIDs = make(map[string]string, len(src.ProviderIDs))
}
for k, v := range src.ProviderIDs {
if v != "" {
if _, exists := dst.ProviderIDs[k]; !exists {
dst.ProviderIDs[k] = v
}
}
}
}
func providerIDMapFromRows(rows []*models.MediaItemProviderID) map[string]string {
if len(rows) == 0 {
return nil
}
m := make(map[string]string, len(rows))
for _, r := range rows {
if r != nil && strings.TrimSpace(r.Provider) != "" && strings.TrimSpace(r.ProviderID) != "" {
m[strings.TrimSpace(r.Provider)] = strings.TrimSpace(r.ProviderID)
}
}
return m
}
+254
View File
@@ -0,0 +1,254 @@
package ebooks
import (
"context"
"errors"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/metadata"
"github.com/Silo-Server/silo-server/internal/models"
)
func TestEbookContentType(t *testing.T) {
if got := ebookContentType(); got != "ebook" {
t.Fatalf("ebookContentType() = %q, want ebook", got)
}
}
func TestFilterEbookPeopleKeepsAuthorsOnly(t *testing.T) {
people := []models.ItemPerson{
{Person: models.Person{Name: "Author One"}, Kind: models.PersonKindAuthor, SortOrder: 7},
{Person: models.Person{Name: "Narrator One"}, Kind: models.PersonKindNarrator, SortOrder: 8},
{Person: models.Person{Name: "Writer One"}, Kind: models.PersonKindWriter, SortOrder: 9},
{Person: models.Person{Name: "Author Two"}, Kind: models.PersonKindAuthor, SortOrder: 10},
}
got := filterEbookPeople(people)
if len(got) != 2 {
t.Fatalf("filtered people len = %d, want 2: %+v", len(got), got)
}
for i, p := range got {
if p.Kind != models.PersonKindAuthor {
t.Fatalf("filtered[%d].Kind = %v, want author", i, p.Kind)
}
if p.SortOrder != i {
t.Fatalf("filtered[%d].SortOrder = %d, want %d", i, p.SortOrder, i)
}
}
if got[0].Person.Name != "Author One" || got[1].Person.Name != "Author Two" {
t.Fatalf("filtered author order = %+v", got)
}
}
func TestEbookEnrichWorkersFromEnv(t *testing.T) {
t.Setenv("SILO_EBOOK_ENRICH_WORKERS", "12")
if got := ebookEnrichWorkers(); got != 12 {
t.Fatalf("ebookEnrichWorkers() = %d, want 12", got)
}
t.Setenv("SILO_EBOOK_ENRICH_WORKERS", "0")
if got := ebookEnrichWorkers(); got != defaultEnrichWorkers {
t.Fatalf("ebookEnrichWorkers() with zero = %d, want default %d", got, defaultEnrichWorkers)
}
t.Setenv("SILO_EBOOK_ENRICH_WORKERS", "999")
if got := ebookEnrichWorkers(); got != defaultEnrichBatchSize {
t.Fatalf("ebookEnrichWorkers() capped = %d, want %d", got, defaultEnrichBatchSize)
}
}
func TestEnricherRunFansOut(t *testing.T) {
const wantWorkers = 4
const itemCount = 16
items := make([]enrichmentItemRow, itemCount)
for i := range items {
items[i] = enrichmentItemRow{ContentID: "test", Title: "t"}
}
var inFlight int32
var maxInFlight int32
var signaled int32
var wg sync.WaitGroup
wg.Add(wantWorkers)
gate := make(chan struct{})
enrich := func(ctx context.Context, item enrichmentItemRow) error {
cur := atomic.AddInt32(&inFlight, 1)
defer atomic.AddInt32(&inFlight, -1)
for {
prev := atomic.LoadInt32(&maxInFlight)
if cur <= prev || atomic.CompareAndSwapInt32(&maxInFlight, prev, cur) {
break
}
}
if atomic.AddInt32(&signaled, 1) <= wantWorkers {
wg.Done()
}
select {
case <-gate:
case <-time.After(2 * time.Second):
}
return nil
}
go func() {
wg.Wait()
close(gate)
}()
e := &Enricher{workers: wantWorkers, batchSize: itemCount}
e.runBatch(context.Background(), items, enrich)
if got := atomic.LoadInt32(&maxInFlight); got < wantWorkers {
t.Errorf("max in-flight = %d, want >= %d", got, wantWorkers)
}
}
func TestCacheRemotePosterCachesProviderURL(t *testing.T) {
cacher := &fakeEbookImageCacher{}
e := &Enricher{imageCacher: cacher}
result := &metadata.MetadataResult{
PosterPath: "https://example.test/book.jpg",
}
e.cacheRemotePoster(context.Background(), "content-1", result)
if cacher.calls != 1 {
t.Fatalf("CacheImage calls = %d, want 1", cacher.calls)
}
if cacher.req.ProviderID != ebookMetadataImageProviderID {
t.Fatalf("ProviderID = %q, want %q", cacher.req.ProviderID, ebookMetadataImageProviderID)
}
if cacher.req.ContentType != "ebooks" || cacher.req.ContentID != "content-1" {
t.Fatalf("cache target = %q/%q", cacher.req.ContentType, cacher.req.ContentID)
}
if result.PosterPath != "ebook-metadata/ebooks/content-1/poster/original.webp" {
t.Fatalf("PosterPath = %q", result.PosterPath)
}
if result.PosterThumbhash != "thumb" {
t.Fatalf("PosterThumbhash = %q", result.PosterThumbhash)
}
}
func TestCacheRemotePosterSkipsNilCacher(t *testing.T) {
e := &Enricher{}
result := &metadata.MetadataResult{
PosterPath: "https://example.test/book.jpg",
}
e.cacheRemotePoster(context.Background(), "content-1", result)
if result.PosterPath != "https://example.test/book.jpg" {
t.Fatalf("PosterPath = %q, want provider URL preserved", result.PosterPath)
}
}
func TestCacheRemotePosterSkipsTypedNilCacher(t *testing.T) {
var cacher *fakeEbookImageCacher
e := &Enricher{imageCacher: cacher}
result := &metadata.MetadataResult{
PosterPath: "https://example.test/book.jpg",
}
e.cacheRemotePoster(context.Background(), "content-1", result)
if result.PosterPath != "https://example.test/book.jpg" {
t.Fatalf("PosterPath = %q, want provider URL preserved", result.PosterPath)
}
}
func TestCacheRemotePosterSkipsAlreadyCachedPath(t *testing.T) {
cacher := &fakeEbookImageCacher{}
e := &Enricher{imageCacher: cacher}
result := &metadata.MetadataResult{
PosterPath: "local/ebooks/content-1/poster/original.webp",
}
e.cacheRemotePoster(context.Background(), "content-1", result)
if cacher.calls != 0 {
t.Fatalf("CacheImage calls = %d, want 0", cacher.calls)
}
if result.PosterPath != "local/ebooks/content-1/poster/original.webp" {
t.Fatalf("PosterPath = %q", result.PosterPath)
}
}
func TestCacheRemotePosterPreservesProviderURLOnCacheError(t *testing.T) {
cacher := &fakeEbookImageCacher{err: errors.New("cache failed")}
e := &Enricher{imageCacher: cacher}
result := &metadata.MetadataResult{
PosterPath: "https://example.test/book.jpg",
}
e.cacheRemotePoster(context.Background(), "content-1", result)
if cacher.calls != 1 {
t.Fatalf("CacheImage calls = %d, want 1", cacher.calls)
}
if result.PosterPath != "https://example.test/book.jpg" {
t.Fatalf("PosterPath = %q, want provider URL preserved", result.PosterPath)
}
}
func TestCacheRemotePosterPreservesProviderURLOnNilCacheResult(t *testing.T) {
cacher := &fakeEbookImageCacher{returnNil: true}
e := &Enricher{imageCacher: cacher}
result := &metadata.MetadataResult{
PosterPath: "https://example.test/book.jpg",
}
e.cacheRemotePoster(context.Background(), "content-1", result)
if cacher.calls != 1 {
t.Fatalf("CacheImage calls = %d, want 1", cacher.calls)
}
if result.PosterPath != "https://example.test/book.jpg" {
t.Fatalf("PosterPath = %q, want provider URL preserved", result.PosterPath)
}
}
func TestMergeEnrichmentProviderIDsKeepsExistingIDs(t *testing.T) {
dst := &metadata.MetadataResult{ProviderIDs: map[string]string{"isbn": "9780306406157"}}
src := &metadata.MetadataResult{ProviderIDs: map[string]string{"isbn": "new", "openlibrary": "OL1M", "empty": ""}}
mergeEnrichmentProviderIDs(dst, src)
if got := dst.ProviderIDs["isbn"]; got != "9780306406157" {
t.Fatalf("isbn = %q, want original", got)
}
if got := dst.ProviderIDs["openlibrary"]; got != "OL1M" {
t.Fatalf("openlibrary = %q, want OL1M", got)
}
if _, exists := dst.ProviderIDs["empty"]; exists {
t.Fatal("empty provider ID should not be merged")
}
}
type fakeEbookImageCacher struct {
calls int
req metadata.CacheImageRequest
err error
returnNil bool
}
func (f *fakeEbookImageCacher) CacheImage(_ context.Context, req metadata.CacheImageRequest) (*metadata.CacheImageResult, error) {
f.calls++
f.req = req
if f.err != nil {
return nil, f.err
}
if f.returnNil {
return nil, nil
}
return &metadata.CacheImageResult{
BasePath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster",
Thumbhash: "thumb",
Ext: ".webp",
}, nil
}
+377
View File
@@ -0,0 +1,377 @@
package scanner
import (
"archive/zip"
"encoding/xml"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const maxEPUBMetadataEntrySize = 8 * 1024 * 1024
var ebookExtensions = map[string]bool{
".epub": true,
".pdf": true,
".mobi": true,
".azw": true,
".azw3": true,
".fb2": true,
}
type parsedEbook struct {
Format string
Title string
Authors []string
Description string
Publisher string
PublishedAt time.Time
Year int
Language string
ISBN string
Series string
SeriesIndex string
Genres []string
PageCount int
Cover *parsedEbookCover
}
type parsedEbookCover struct {
ContentType string
Bytes []byte
}
func SupportsEbookFile(filePath string) bool {
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
}
func parseEbookFile(path string) (book parsedEbook, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic parsing ebook %s: %v", path, r)
}
}()
switch strings.ToLower(filepath.Ext(path)) {
case ".epub":
book, err = parseEbookEPUB(path)
case ".fb2":
book, err = parseEbookFB2(path)
case ".pdf", ".mobi", ".azw", ".azw3":
book = parsedEbook{Format: strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")}
default:
err = fmt.Errorf("unsupported ebook format: %s", filepath.Ext(path))
}
book.sanitize()
return book, err
}
func (b *parsedEbook) sanitize() {
b.Format = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(b.Format)), ".")
b.Title = strings.TrimSpace(b.Title)
b.Description = strings.TrimSpace(b.Description)
b.Publisher = strings.TrimSpace(b.Publisher)
b.Language = strings.TrimSpace(b.Language)
b.ISBN = normalizeEbookISBN(b.ISBN)
b.Series = strings.TrimSpace(b.Series)
b.SeriesIndex = strings.TrimSpace(b.SeriesIndex)
b.Authors = uniqueTrimmedStrings(b.Authors)
b.Genres = uniqueTrimmedStrings(b.Genres)
if b.PageCount < 0 {
b.PageCount = 0
}
if b.Year == 0 && !b.PublishedAt.IsZero() {
b.Year = b.PublishedAt.Year()
}
}
func uniqueTrimmedStrings(values []string) []string {
seen := map[string]struct{}{}
var out []string
for _, value := range values {
trimmed := strings.TrimSpace(value)
key := strings.ToLower(trimmed)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, trimmed)
}
return out
}
func normalizeEbookISBN(value string) string {
value = strings.ToUpper(strings.TrimSpace(value))
for _, prefix := range []string{"ISBN-13", "ISBN-10", "ISBN"} {
if strings.HasPrefix(value, prefix) {
value = strings.TrimSpace(strings.TrimPrefix(value, prefix))
value = strings.TrimLeft(value, ": ")
break
}
}
var out strings.Builder
for _, r := range value {
if r >= '0' && r <= '9' {
out.WriteRune(r)
continue
}
if r == 'X' {
out.WriteRune(r)
}
}
candidate := out.String()
switch len(candidate) {
case 10:
if validISBN10(candidate) {
return candidate
}
case 13:
if validISBN13(candidate) {
return candidate
}
}
return ""
}
func validISBN10(candidate string) bool {
if len(candidate) != 10 {
return false
}
sum := 0
for i, r := range candidate {
value := 0
switch {
case r >= '0' && r <= '9':
value = int(r - '0')
case r == 'X' && i == 9:
value = 10
default:
return false
}
sum += value * (10 - i)
}
return sum%11 == 0
}
func validISBN13(candidate string) bool {
if len(candidate) != 13 {
return false
}
sum := 0
for i, r := range candidate {
if r < '0' || r > '9' {
return false
}
value := int(r - '0')
if i%2 == 1 {
value *= 3
}
sum += value
}
return sum%10 == 0
}
func parseEbookEPUB(path string) (parsedEbook, error) {
book := parsedEbook{Format: "epub"}
reader, err := zip.OpenReader(path)
if err != nil {
return book, err
}
defer reader.Close()
container, err := readEPUBZipEntry(&reader.Reader, "META-INF/container.xml")
if err != nil {
return book, err
}
opfPath, err := epubOPFPath(container)
if err != nil {
return book, err
}
opf, err := readEPUBZipEntry(&reader.Reader, opfPath)
if err != nil {
return book, err
}
if err := parseEPUBOPFMetadata(opf, &book); err != nil {
return book, err
}
return book, nil
}
func parseEbookFB2(path string) (parsedEbook, error) {
book := parsedEbook{Format: "fb2"}
file, err := os.Open(path)
if err != nil {
return book, err
}
defer file.Close()
var fb2 struct {
Description struct {
TitleInfo struct {
Genres []string `xml:"genre"`
Authors []struct {
FirstName string `xml:"first-name"`
MiddleName string `xml:"middle-name"`
LastName string `xml:"last-name"`
Nickname string `xml:"nickname"`
} `xml:"author"`
BookTitle string `xml:"book-title"`
Lang string `xml:"lang"`
Date struct {
Value string `xml:"value,attr"`
Text string `xml:",chardata"`
} `xml:"date"`
Sequences []struct {
Name string `xml:"name,attr"`
Number string `xml:"number,attr"`
} `xml:"sequence"`
} `xml:"title-info"`
PublishInfo struct {
ISBN string `xml:"isbn"`
Publisher string `xml:"publisher"`
Year string `xml:"year"`
} `xml:"publish-info"`
} `xml:"description"`
}
if err := xml.NewDecoder(file).Decode(&fb2); err != nil {
return book, err
}
book.Title = fb2.Description.TitleInfo.BookTitle
book.Language = fb2.Description.TitleInfo.Lang
book.Genres = fb2.Description.TitleInfo.Genres
if t, ok := parseEbookDate(firstNonEmpty(fb2.Description.TitleInfo.Date.Value, fb2.Description.TitleInfo.Date.Text)); ok {
book.PublishedAt = t
}
for _, author := range fb2.Description.TitleInfo.Authors {
name := strings.Join(uniqueTrimmedStrings([]string{
author.FirstName,
author.MiddleName,
author.LastName,
}), " ")
if name == "" {
name = author.Nickname
}
book.Authors = append(book.Authors, name)
}
if len(fb2.Description.TitleInfo.Sequences) > 0 {
book.Series = fb2.Description.TitleInfo.Sequences[0].Name
book.SeriesIndex = fb2.Description.TitleInfo.Sequences[0].Number
}
book.ISBN = fb2.Description.PublishInfo.ISBN
book.Publisher = fb2.Description.PublishInfo.Publisher
if year, err := strconv.Atoi(strings.TrimSpace(fb2.Description.PublishInfo.Year)); err == nil {
book.Year = year
}
return book, nil
}
func readEPUBZipEntry(reader *zip.Reader, name string) ([]byte, error) {
for _, file := range reader.File {
if file.Name != name {
continue
}
entry, err := file.Open()
if err != nil {
return nil, err
}
defer entry.Close()
limited := io.LimitReader(entry, maxEPUBMetadataEntrySize+1)
data, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if len(data) > maxEPUBMetadataEntrySize {
return nil, fmt.Errorf("epub entry too large: %s", name)
}
return data, nil
}
return nil, fmt.Errorf("epub entry not found: %s", name)
}
func epubOPFPath(container []byte) (string, error) {
var parsed struct {
Rootfiles []struct {
FullPath string `xml:"full-path,attr"`
} `xml:"rootfiles>rootfile"`
}
if err := xml.Unmarshal(container, &parsed); err != nil {
return "", err
}
for _, rootfile := range parsed.Rootfiles {
if strings.TrimSpace(rootfile.FullPath) != "" {
return rootfile.FullPath, nil
}
}
return "", fmt.Errorf("epub container has no rootfile")
}
func parseEPUBOPFMetadata(opf []byte, book *parsedEbook) error {
var parsed struct {
Metadata struct {
Titles []string `xml:"title"`
Creators []string `xml:"creator"`
Identifiers []string `xml:"identifier"`
Publisher string `xml:"publisher"`
Dates []string `xml:"date"`
Language string `xml:"language"`
Subjects []string `xml:"subject"`
Descriptions []string `xml:"description"`
Meta []struct {
Name string `xml:"name,attr"`
Property string `xml:"property,attr"`
Content string `xml:"content,attr"`
Value string `xml:",chardata"`
} `xml:"meta"`
} `xml:"metadata"`
}
if err := xml.Unmarshal(opf, &parsed); err != nil {
return err
}
book.Title = firstNonEmpty(parsed.Metadata.Titles...)
book.Authors = append(book.Authors, parsed.Metadata.Creators...)
book.Publisher = parsed.Metadata.Publisher
book.Language = parsed.Metadata.Language
book.Genres = append(book.Genres, parsed.Metadata.Subjects...)
book.Description = firstNonEmpty(parsed.Metadata.Descriptions...)
for _, identifier := range parsed.Metadata.Identifiers {
if isbn := normalizeEbookISBN(identifier); isbn != "" {
book.ISBN = isbn
break
}
}
for _, date := range parsed.Metadata.Dates {
if t, ok := parseEbookDate(date); ok {
book.PublishedAt = t
book.Year = t.Year()
break
}
}
for _, meta := range parsed.Metadata.Meta {
name := strings.ToLower(strings.TrimSpace(firstNonEmpty(meta.Name, meta.Property)))
value := strings.TrimSpace(firstNonEmpty(meta.Content, meta.Value))
switch name {
case "calibre:series", "belongs-to-collection":
book.Series = value
case "calibre:series_index", "group-position":
book.SeriesIndex = value
}
}
return nil
}
func parseEbookDate(value string) (time.Time, bool) {
value = strings.TrimSpace(value)
for _, layout := range []string{"2006-01-02", "2006-01", "2006"} {
if t, err := time.Parse(layout, value); err == nil {
return t, true
}
}
return time.Time{}, false
}
+686
View File
@@ -0,0 +1,686 @@
package scanner
import (
"context"
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Silo-Server/silo-server/internal/idgen"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/titleutil"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type ebookSQLExecutor interface {
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
}
type filesystemMediaItemReader interface {
GetByIDs(ctx context.Context, ids []string) ([]*models.MediaItem, error)
}
func ebookScanWorkers() int {
if v := os.Getenv("SILO_EBOOK_SCAN_WORKERS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return audiobookScanWorkers()
}
func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolder) error {
if s == nil || folder == nil {
return fmt.Errorf("ScanEbookFolder: nil scanner or folder")
}
var candidates []string
for _, root := range folder.Paths {
if err := ctx.Err(); err != nil {
return err
}
walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
if err := ctx.Err(); err != nil {
return err
}
if walkErr != nil {
slog.Warn("ebook scan: walk error", "path", path, "error", walkErr)
return nil
}
if d.IsDir() {
return nil
}
if SupportsEbookFile(path) {
candidates = append(candidates, path)
}
return nil
})
if walkErr != nil {
slog.Warn("ebook scan: walk root failed", "root", root, "error", walkErr)
}
}
if len(candidates) == 0 {
return nil
}
workers := ebookScanWorkers()
slog.Info("ebook scan: starting",
"folder_id", folder.ID,
"candidates", len(candidates),
"workers", workers,
)
ch := make(chan string, workers*2)
var (
wg sync.WaitGroup
processed int64
failed int64
skipped int64
failMu sync.Mutex
failures []error
cancelErr error
)
start := time.Now()
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for path := range ch {
if ctx.Err() != nil {
return
}
if err := s.reconcileEbookFile(ctx, folder, path, &skipped); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
failMu.Lock()
if cancelErr == nil {
cancelErr = err
}
failMu.Unlock()
return
}
atomic.AddInt64(&failed, 1)
failMu.Lock()
failures = append(failures, fmt.Errorf("%s: %w", path, err))
failMu.Unlock()
slog.Warn("ebook scan: file failed",
"folder_id", folder.ID,
"path", path,
"error", err,
)
}
n := atomic.AddInt64(&processed, 1)
if n%500 == 0 {
slog.Info("ebook scan: progress",
"folder_id", folder.ID,
"processed", n,
"failed", atomic.LoadInt64(&failed),
"skipped", atomic.LoadInt64(&skipped),
"total", len(candidates),
"elapsed_sec", int(time.Since(start).Seconds()),
)
}
}
}()
}
for _, p := range candidates {
select {
case ch <- p:
case <-ctx.Done():
close(ch)
wg.Wait()
return ctx.Err()
}
}
close(ch)
wg.Wait()
if err := ctx.Err(); err != nil {
return err
}
if cancelErr != nil {
return cancelErr
}
slog.Info("ebook scan: completed",
"folder_id", folder.ID,
"processed", atomic.LoadInt64(&processed),
"failed", atomic.LoadInt64(&failed),
"skipped", atomic.LoadInt64(&skipped),
"elapsed_sec", int(time.Since(start).Seconds()),
)
if processedCount := atomic.LoadInt64(&processed); processedCount > 0 {
failedCount := atomic.LoadInt64(&failed)
skippedCount := atomic.LoadInt64(&skipped)
if failedCount > 0 && skippedCount == 0 && failedCount == processedCount {
return fmt.Errorf("ebook scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
}
}
return nil
}
func (s *Scanner) reconcileEbookFile(ctx context.Context, folder *models.MediaFolder, filePath string, skipped *int64) error {
info, err := os.Stat(filePath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("stat ebook file %s: %w", filePath, err)
}
size := info.Size()
modifiedAt := normalizeFileModifiedAt(info.ModTime())
isUnchanged, skipErr := s.ebookFileShouldSkip(ctx, folder, filePath, size, modifiedAt)
if skipErr != nil {
slog.Warn("ebook scan: skip-check failed, falling through",
"folder_id", folder.ID,
"path", filePath,
"error", skipErr,
)
} else if isUnchanged {
atomic.AddInt64(skipped, 1)
return nil
}
parsed, err := parseEbookFile(filePath)
if err != nil {
return fmt.Errorf("parse ebook file %s: %w", filePath, err)
}
if parsed.Title == "" {
parsed.Title = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
}
contentID, err := s.upsertEbookMediaItem(ctx, folder.ID, filePath, &parsed)
if err != nil {
return fmt.Errorf("upsert ebook item: %w", err)
}
if err := s.upsertEbookMediaFile(ctx, folder, contentID, filePath, size, modifiedAt, &parsed); err != nil {
return fmt.Errorf("upsert ebook file: %w", err)
}
if err := s.upsertEbookPeople(ctx, contentID, &parsed); err != nil {
return fmt.Errorf("upsert ebook people: %w", err)
}
if err := s.upsertEbookSeries(ctx, contentID, &parsed); err != nil {
return fmt.Errorf("upsert ebook series: %w", err)
}
if err := insertEbookLibraryMembership(ctx, s.fileRepo.Pool(), contentID, folder.ID); err != nil {
return fmt.Errorf("upsert ebook library membership: %w", err)
}
if parsed.ISBN != "" {
if err := insertEbookISBNProviderID(ctx, s.fileRepo.Pool(), contentID, parsed.ISBN); err != nil {
return fmt.Errorf("upsert ebook ISBN provider id: %w", err)
}
}
slog.Info("ebook scan: indexed",
"folder_id", folder.ID,
"content_id", contentID,
"title", parsed.Title,
"authors", parsed.Authors,
"path", filePath,
)
return nil
}
func (s *Scanner) ebookFileShouldSkip(ctx context.Context, folder *models.MediaFolder, filePath string, size int64, modifiedAt time.Time) (bool, error) {
if s.fileRepo == nil || s.itemRepo == nil {
return false, nil
}
existing, err := s.fileRepo.ListByObservedRootPath(ctx, folder.ID, filePath)
if err != nil {
return false, fmt.Errorf("list existing files: %w", err)
}
if len(existing) != 1 {
return false, nil
}
mf := existing[0]
if mf.FilePath != filePath || mf.FileSize != size || mf.FileModifiedAt == nil || !sameFileModifiedAt(mf.FileModifiedAt, modifiedAt) {
return false, nil
}
if mf.ContentID == "" {
return false, nil
}
statuses, err := s.itemRepo.GetStatusByIDs(ctx, []string{mf.ContentID})
if err != nil {
return false, fmt.Errorf("get item status: %w", err)
}
return !strings.EqualFold(strings.TrimSpace(statuses[mf.ContentID]), "unmatched"), nil
}
func (s *Scanner) upsertEbookMediaItem(ctx context.Context, folderID int, filePath string, book *parsedEbook) (string, error) {
if s.itemRepo == nil {
return "", fmt.Errorf("itemRepo not configured on Scanner")
}
if s.fileRepo == nil {
return "", fmt.Errorf("fileRepo not configured on Scanner")
}
existingID, err := s.fileRepo.FindContentIDByRootPath(ctx, folderID, filePath, "ebook")
if err != nil {
return "", fmt.Errorf("find ebook by root path: %w", err)
}
if existingID != "" {
if err := updateExistingEbookMediaItem(ctx, s.itemRepo, s.itemRepo, existingID, book); err != nil {
return "", err
}
return existingID, nil
}
if existing := s.findEbookByFilePath(ctx, filePath); existing != nil {
applyEbookToMediaItem(existing, book)
if existing.SortTitle == "" {
existing.SortTitle = titleutil.DeriveDefaultSortTitle(existing.Title)
}
if err := s.itemRepo.Upsert(ctx, existing); err != nil {
return "", err
}
return existing.ContentID, nil
}
return resolveEbookMediaItem(ctx, s.fileRepo, s.itemRepo, folderID, filePath, book)
}
func resolveEbookMediaItem(
ctx context.Context,
rootFinder filesystemRootContentFinder,
itemWriter filesystemMediaItemWriter,
folderID int,
filePath string,
book *parsedEbook,
) (string, error) {
if rootFinder == nil {
return "", fmt.Errorf("root content finder not configured")
}
if itemWriter == nil {
return "", fmt.Errorf("media item writer not configured")
}
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, filePath, "ebook")
if err != nil {
return "", fmt.Errorf("find ebook by root path: %w", err)
}
if existingID != "" {
return existingID, nil
}
cleanTitle := strings.TrimSpace(book.Title)
if cleanTitle == "" {
cleanTitle = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
}
return createEbookMediaItem(ctx, itemWriter, book, cleanTitle)
}
func createEbookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedEbook, cleanTitle string) (string, error) {
id, err := idgen.NextID()
if err != nil {
return "", fmt.Errorf("generate content_id: %w", err)
}
item := &models.MediaItem{
ContentID: id,
Type: "ebook",
Title: cleanTitle,
SortTitle: titleutil.DeriveDefaultSortTitle(cleanTitle),
Year: book.Year,
}
applyEbookToMediaItem(item, book)
if item.SortTitle == "" {
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
}
if err := itemWriter.Upsert(ctx, item); err != nil {
return "", err
}
return id, nil
}
func updateExistingEbookMediaItem(ctx context.Context, itemReader filesystemMediaItemReader, itemWriter filesystemMediaItemWriter, contentID string, book *parsedEbook) error {
if itemReader == nil {
return fmt.Errorf("media item reader not configured")
}
if itemWriter == nil {
return fmt.Errorf("media item writer not configured")
}
items, err := itemReader.GetByIDs(ctx, []string{contentID})
if err != nil {
return fmt.Errorf("get ebook media item %s: %w", contentID, err)
}
if len(items) == 0 || items[0] == nil {
return fmt.Errorf("ebook media item %s not found", contentID)
}
item := items[0]
applyEbookToMediaItem(item, book)
if item.SortTitle == "" {
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
}
return itemWriter.Upsert(ctx, item)
}
func applyEbookToMediaItem(item *models.MediaItem, book *parsedEbook) {
item.Type = "ebook"
if title := strings.TrimSpace(book.Title); title != "" {
item.Title = title
}
if book.Year > 0 {
item.Year = book.Year
}
if book.Description != "" && item.Overview == "" {
item.Overview = book.Description
}
if book.Publisher != "" {
item.Studios = mergeUniqueStrings(item.Studios, []string{book.Publisher})
}
if len(book.Genres) > 0 {
item.Genres = mergeUniqueStrings(item.Genres, book.Genres)
}
if !book.PublishedAt.IsZero() && item.ReleaseDate == nil {
rd := book.PublishedAt.UTC().Format("2006-01-02")
item.ReleaseDate = &rd
}
if book.Language != "" && item.OriginalLanguage == "" {
item.OriginalLanguage = book.Language
}
}
func (s *Scanner) findEbookByFilePath(ctx context.Context, filePath string) *models.MediaItem {
if s.fileRepo == nil {
return nil
}
var existingID string
err := s.fileRepo.Pool().QueryRow(ctx, `
SELECT mf.content_id
FROM media_files mf
JOIN media_items mi ON mi.content_id = mf.content_id
WHERE mf.file_path = $1
AND mi.type = 'ebook'
LIMIT 1
`, filePath).Scan(&existingID)
if err != nil || existingID == "" {
return nil
}
items, err := s.itemRepo.GetByIDs(ctx, []string{existingID})
if err != nil || len(items) == 0 {
return nil
}
return items[0]
}
func (s *Scanner) upsertEbookMediaFile(ctx context.Context, folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) error {
mf := buildEbookMediaFile(folder, contentID, filePath, size, modifiedAt, book)
if _, err := s.fileRepo.Upsert(ctx, mf); err != nil {
return fmt.Errorf("upsert media file %s: %w", filePath, err)
}
return nil
}
func buildEbookMediaFile(folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) models.MediaFile {
return models.MediaFile{
ContentID: contentID,
MediaFolderID: folder.ID,
CanonicalRootPath: filePath,
ObservedRootPath: filePath,
ContentGroupKey: contentID,
GroupKeyVersion: 1,
BaseTitle: book.Title,
BaseYear: book.Year,
BaseType: "ebook",
IdentityConfidence: ebookIdentityConfidence(book),
FilePath: filePath,
FileSize: size,
FileModifiedAt: &modifiedAt,
Container: book.Format,
ProbeSource: "local",
}
}
func insertEbookLibraryMembership(ctx context.Context, exec ebookSQLExecutor, contentID string, folderID int) error {
_, err := exec.Exec(ctx, `
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
VALUES ($1, $2, NOW())
ON CONFLICT (content_id, media_folder_id) DO NOTHING
`, contentID, folderID)
return err
}
func insertEbookISBNProviderID(ctx context.Context, exec ebookSQLExecutor, contentID string, isbn string) error {
_, err := exec.Exec(ctx, `
INSERT INTO media_item_provider_ids (content_id, provider, provider_id, item_type)
VALUES ($1, 'isbn', $2, 'ebook')
ON CONFLICT DO NOTHING
`, contentID, isbn)
return err
}
type ebookCredit struct {
Name string
Kind models.PersonKind
}
func ebookPeopleCreditsEqual(existing []models.ItemPerson, desired []ebookCredit) bool {
existingAuthors := make([]models.ItemPerson, 0, len(existing))
for _, p := range existing {
if p.Kind == models.PersonKindAuthor {
existingAuthors = append(existingAuthors, p)
}
}
if len(existingAuthors) != len(desired) {
return false
}
type key struct {
name string
kind models.PersonKind
}
have := make(map[key]struct{}, len(existingAuthors))
for _, p := range existingAuthors {
have[key{strings.ToLower(strings.TrimSpace(p.Person.Name)), p.Kind}] = struct{}{}
}
for _, d := range desired {
k := key{strings.ToLower(strings.TrimSpace(d.Name)), d.Kind}
if _, ok := have[k]; !ok {
return false
}
}
return true
}
type ebookResolvedAuthor struct {
ID int64
Name string
}
func mergeEbookPeople(existing []models.ItemPerson, authors []ebookResolvedAuthor) []models.ItemPerson {
people := make([]models.ItemPerson, 0, len(existing)+len(authors))
for _, p := range existing {
if p.Kind == models.PersonKindAuthor {
continue
}
p.SortOrder = len(people)
people = append(people, p)
}
for _, author := range authors {
people = append(people, models.ItemPerson{
Person: models.Person{ID: author.ID, Name: author.Name},
Kind: models.PersonKindAuthor,
SortOrder: len(people),
})
}
return people
}
func ebookPeopleForReplace(existing []models.ItemPerson, getErr error, authors []ebookResolvedAuthor) ([]models.ItemPerson, error) {
if getErr != nil {
return nil, fmt.Errorf("get ebook people: %w", getErr)
}
return mergeEbookPeople(existing, authors), nil
}
func (s *Scanner) upsertEbookPeople(ctx context.Context, contentID string, book *parsedEbook) error {
if s.personRepo == nil {
return fmt.Errorf("personRepo not configured on Scanner")
}
if s.itemRepo == nil {
return fmt.Errorf("itemRepo not configured on Scanner")
}
var desired []ebookCredit
for _, author := range book.Authors {
if name := strings.TrimSpace(author); name != "" {
desired = append(desired, ebookCredit{Name: name, Kind: models.PersonKindAuthor})
}
}
if len(desired) == 0 {
return nil
}
existing, err := s.itemRepo.GetPeople(ctx, contentID)
if err != nil {
return fmt.Errorf("get ebook people: %w", err)
}
if ebookPeopleCreditsEqual(existing, desired) {
return nil
}
authors := make([]ebookResolvedAuthor, 0, len(desired))
for _, c := range desired {
personID, err := s.personRepo.FindOrCreate(ctx, models.Person{Name: c.Name})
if err != nil {
return fmt.Errorf("find-or-create person %q: %w", c.Name, err)
}
authors = append(authors, ebookResolvedAuthor{ID: personID, Name: c.Name})
}
people, err := ebookPeopleForReplace(existing, nil, authors)
if err != nil {
return err
}
return s.itemRepo.ReplacePeople(ctx, contentID, people)
}
func (s *Scanner) upsertEbookSeries(ctx context.Context, contentID string, book *parsedEbook) error {
if s == nil {
return fmt.Errorf("Scanner not configured")
}
if s.fileRepo == nil {
return fmt.Errorf("fileRepo not configured on Scanner")
}
var currentName *string
var currentIdx *float64
err := s.fileRepo.Pool().QueryRow(ctx, `
SELECT series_name, series_index FROM ebook_series WHERE content_id = $1
`, contentID).Scan(&currentName, &currentIdx)
plan, err := planEbookSeriesWrite(book, currentName, currentIdx, err)
if err != nil {
return err
}
switch plan.Kind {
case ebookSeriesWriteNone:
return nil
case ebookSeriesWriteDelete:
if _, delErr := s.fileRepo.Pool().Exec(ctx,
`DELETE FROM ebook_series WHERE content_id = $1`, contentID); delErr != nil {
return fmt.Errorf("delete ebook_series row: %w", delErr)
}
return nil
case ebookSeriesWriteUpsert:
var idx any
if plan.Index != nil {
idx = *plan.Index
}
if _, err := s.fileRepo.Pool().Exec(ctx, `
INSERT INTO ebook_series (content_id, series_name, series_index, updated_at)
VALUES ($1, $2, $3, NOW())
ON CONFLICT (content_id) DO UPDATE SET
series_name = EXCLUDED.series_name,
series_index = EXCLUDED.series_index,
updated_at = NOW()
`, contentID, plan.Name, idx); err != nil {
return fmt.Errorf("upsert ebook_series row: %w", err)
}
return nil
default:
return fmt.Errorf("unknown ebook_series write kind: %d", plan.Kind)
}
}
func ebookSeriesDesired(book *parsedEbook) (string, *float64) {
if book == nil {
return "", nil
}
return strings.TrimSpace(book.Series), parseSeriesIndex(book.SeriesIndex)
}
type ebookSeriesWriteKind int
const (
ebookSeriesWriteNone ebookSeriesWriteKind = iota
ebookSeriesWriteDelete
ebookSeriesWriteUpsert
)
type ebookSeriesWritePlan struct {
Kind ebookSeriesWriteKind
Name string
Index *float64
}
func planEbookSeriesWrite(book *parsedEbook, currentName *string, currentIdx *float64, queryErr error) (ebookSeriesWritePlan, error) {
if queryErr != nil {
if !errors.Is(queryErr, pgx.ErrNoRows) {
return ebookSeriesWritePlan{}, fmt.Errorf("query ebook_series: %w", queryErr)
}
currentName = nil
currentIdx = nil
}
desiredName, desiredIdx := ebookSeriesDesired(book)
if desiredName == "" {
if currentName == nil {
return ebookSeriesWritePlan{Kind: ebookSeriesWriteNone}, nil
}
return ebookSeriesWritePlan{Kind: ebookSeriesWriteDelete}, nil
}
if currentName != nil && *currentName == desiredName && floatPtrEqual(currentIdx, desiredIdx) {
return ebookSeriesWritePlan{Kind: ebookSeriesWriteNone}, nil
}
return ebookSeriesWritePlan{
Kind: ebookSeriesWriteUpsert,
Name: desiredName,
Index: desiredIdx,
}, nil
}
func ebookIdentityConfidence(book *parsedEbook) string {
if book == nil {
return "low"
}
score := 0
if book.Title != "" {
score++
}
if len(book.Authors) > 0 {
score++
}
if book.Year > 0 {
score++
}
if book.ISBN != "" {
score++
}
switch {
case score >= 4:
return "high"
case score > 0:
return "medium"
default:
return "low"
}
}
+590
View File
@@ -0,0 +1,590 @@
package scanner
import (
"archive/zip"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type recordingEbookExecutor struct {
queries []string
args [][]any
}
func (r *recordingEbookExecutor) Exec(_ context.Context, query string, args ...any) (pgconn.CommandTag, error) {
r.queries = append(r.queries, query)
r.args = append(r.args, append([]any(nil), args...))
return pgconn.CommandTag{}, nil
}
type fakeFilesystemItemReader struct {
items []*models.MediaItem
err error
}
func (f *fakeFilesystemItemReader) GetByIDs(_ context.Context, _ []string) ([]*models.MediaItem, error) {
return f.items, f.err
}
func TestSupportsEbookFile(t *testing.T) {
cases := []struct {
path string
want bool
}{
{"book.epub", true},
{"book.PDF", true},
{"book.mobi", true},
{"book.azw", true},
{"book.azw3", true},
{"book.FB2", true},
{"book.mp3", false},
{"movie.mkv", false},
}
for _, tc := range cases {
if got := SupportsEbookFile(tc.path); got != tc.want {
t.Errorf("SupportsEbookFile(%q) = %v, want %v", tc.path, got, tc.want)
}
}
}
func TestNormalizeEbookISBNHandlesCommonLabels(t *testing.T) {
cases := []struct {
in string
want string
}{
{"ISBN-13: 978-0-306-40615-7", "9780306406157"},
{"ISBN-10: 0-9752298-0-x", "097522980X"},
}
for _, tc := range cases {
if got := normalizeEbookISBN(tc.in); got != tc.want {
t.Errorf("normalizeEbookISBN(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
func TestParseEbookEPUBMetadata(t *testing.T) {
path := writeTestEPUB(t, []string{"ISBN: 978-0-306-40615-7"})
got, err := parseEbookFile(path)
if err != nil {
t.Fatalf("parseEbookFile: %v", err)
}
if got.Format != "epub" {
t.Fatalf("Format = %q, want epub", got.Format)
}
if got.Title != "The Test Ebook" {
t.Fatalf("Title = %q, want The Test Ebook", got.Title)
}
if strings.Join(got.Authors, ", ") != "Ada Writer, Ben Author" {
t.Fatalf("Authors = %v, want Ada Writer and Ben Author", got.Authors)
}
if got.ISBN != "9780306406157" {
t.Fatalf("ISBN = %q, want normalized ISBN", got.ISBN)
}
if got.Year != 2024 {
t.Fatalf("Year = %d, want 2024", got.Year)
}
if got.Publisher != "Silo Press" {
t.Fatalf("Publisher = %q, want Silo Press", got.Publisher)
}
wantPublishedAt := time.Date(2024, 3, 10, 0, 0, 0, 0, time.UTC)
if !got.PublishedAt.Equal(wantPublishedAt) {
t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, wantPublishedAt)
}
if got.Language != "en" {
t.Fatalf("Language = %q, want en", got.Language)
}
if strings.Join(got.Genres, ", ") != "Fiction, Adventure" {
t.Fatalf("Genres = %v, want Fiction and Adventure", got.Genres)
}
if got.Series != "Test Series" || got.SeriesIndex != "2" {
t.Fatalf("Series = %q/%q, want Test Series/2", got.Series, got.SeriesIndex)
}
}
func TestParseEbookEPUBSkipsUUIDIdentifierBeforeISBN(t *testing.T) {
path := writeTestEPUB(t, []string{
"550e8400-e29b-41d4-a716-446655440000",
"ISBN: 978-0-306-40615-7",
})
got, err := parseEbookFile(path)
if err != nil {
t.Fatalf("parseEbookFile: %v", err)
}
if got.ISBN != "9780306406157" {
t.Fatalf("ISBN = %q, want normalized ISBN after skipping UUID", got.ISBN)
}
}
func TestReadEPUBZipEntryRejectsOversizedEntry(t *testing.T) {
path := filepath.Join(t.TempDir(), "large.epub")
file, err := os.Create(path)
if err != nil {
t.Fatalf("create epub: %v", err)
}
zw := zip.NewWriter(file)
w, err := zw.Create("OPS/content.opf")
if err != nil {
t.Fatalf("create zip entry: %v", err)
}
if _, err := w.Write([]byte(strings.Repeat("x", maxEPUBMetadataEntrySize+1))); err != nil {
t.Fatalf("write zip entry: %v", err)
}
if err := zw.Close(); err != nil {
t.Fatalf("close zip: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("close file: %v", err)
}
reader, err := zip.OpenReader(path)
if err != nil {
t.Fatalf("open zip: %v", err)
}
defer reader.Close()
_, err = readEPUBZipEntry(&reader.Reader, "OPS/content.opf")
if err == nil || !strings.Contains(err.Error(), "epub entry too large") {
t.Fatalf("readEPUBZipEntry error = %v, want size limit error", err)
}
}
func TestParseEbookFB2Metadata(t *testing.T) {
path := filepath.Join(t.TempDir(), "book.fb2")
if err := os.WriteFile(path, []byte(`<?xml version="1.0" encoding="UTF-8"?>
<FictionBook>
<description>
<title-info>
<genre>science fiction</genre>
<author><first-name>Ada</first-name><last-name>Writer</last-name></author>
<book-title>FB2 Test Ebook</book-title>
<date value="2022-04-05">5 April 2022</date>
<lang>en</lang>
<sequence name="FB2 Series" number="3"/>
</title-info>
<publish-info>
<publisher>FB2 Press</publisher>
<year>2022</year>
<isbn>978-0-306-40615-7</isbn>
</publish-info>
</description>
</FictionBook>`), 0o644); err != nil {
t.Fatalf("write fb2: %v", err)
}
got, err := parseEbookFile(path)
if err != nil {
t.Fatalf("parseEbookFile: %v", err)
}
if got.Format != "fb2" || got.Title != "FB2 Test Ebook" {
t.Fatalf("Format/Title = %q/%q, want fb2/FB2 Test Ebook", got.Format, got.Title)
}
if strings.Join(got.Authors, ", ") != "Ada Writer" {
t.Fatalf("Authors = %v, want Ada Writer", got.Authors)
}
if got.ISBN != "9780306406157" {
t.Fatalf("ISBN = %q, want normalized ISBN", got.ISBN)
}
if got.Publisher != "FB2 Press" {
t.Fatalf("Publisher = %q, want FB2 Press", got.Publisher)
}
if got.Year != 2022 {
t.Fatalf("Year = %d, want 2022", got.Year)
}
wantPublishedAt := time.Date(2022, 4, 5, 0, 0, 0, 0, time.UTC)
if !got.PublishedAt.Equal(wantPublishedAt) {
t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, wantPublishedAt)
}
if got.Language != "en" {
t.Fatalf("Language = %q, want en", got.Language)
}
if strings.Join(got.Genres, ", ") != "science fiction" {
t.Fatalf("Genres = %v, want science fiction", got.Genres)
}
if got.Series != "FB2 Series" || got.SeriesIndex != "3" {
t.Fatalf("Series = %q/%q, want FB2 Series/3", got.Series, got.SeriesIndex)
}
}
func TestEbookSeriesDesiredParsesIndex(t *testing.T) {
name, idx := ebookSeriesDesired(&parsedEbook{
Series: " The Expanse ",
SeriesIndex: "2 of 9",
})
if name != "The Expanse" {
t.Fatalf("series name = %q, want The Expanse", name)
}
if idx == nil || *idx != 2 {
t.Fatalf("series index = %v, want 2", idx)
}
}
func TestUpsertEbookSeriesNilScannerReturnsError(t *testing.T) {
var s *Scanner
if err := s.upsertEbookSeries(context.Background(), "content-1", &parsedEbook{Series: "Series"}); err == nil {
t.Fatal("upsertEbookSeries nil scanner error = nil, want error")
}
}
func TestUpsertEbookSeriesNilFileRepoReturnsError(t *testing.T) {
s := &Scanner{}
if err := s.upsertEbookSeries(context.Background(), "content-1", &parsedEbook{Series: "Series"}); err == nil {
t.Fatal("upsertEbookSeries nil fileRepo error = nil, want error")
}
}
func TestPlanEbookSeriesWriteInsertsWhenRowAbsent(t *testing.T) {
plan, err := planEbookSeriesWrite(&parsedEbook{Series: " Series ", SeriesIndex: "2 of 9"}, nil, nil, pgx.ErrNoRows)
if err != nil {
t.Fatalf("planEbookSeriesWrite: %v", err)
}
if plan.Kind != ebookSeriesWriteUpsert || plan.Name != "Series" {
t.Fatalf("plan = %+v, want upsert Series", plan)
}
if plan.Index == nil || *plan.Index != 2 {
t.Fatalf("index = %v, want 2", plan.Index)
}
}
func TestPlanEbookSeriesWriteBlankSeriesDeletesExistingAndSkipsAbsent(t *testing.T) {
currentName := "Series"
plan, err := planEbookSeriesWrite(&parsedEbook{Series: " "}, &currentName, nil, nil)
if err != nil {
t.Fatalf("planEbookSeriesWrite delete: %v", err)
}
if plan.Kind != ebookSeriesWriteDelete {
t.Fatalf("blank existing plan = %+v, want delete", plan)
}
plan, err = planEbookSeriesWrite(&parsedEbook{Series: ""}, nil, nil, pgx.ErrNoRows)
if err != nil {
t.Fatalf("planEbookSeriesWrite absent: %v", err)
}
if plan.Kind != ebookSeriesWriteNone {
t.Fatalf("blank absent plan = %+v, want none", plan)
}
}
func TestPlanEbookSeriesWriteSkipsIdenticalRow(t *testing.T) {
currentName := "Series"
currentIdx := 3.5
plan, err := planEbookSeriesWrite(&parsedEbook{Series: "Series", SeriesIndex: "3.5"}, &currentName, &currentIdx, nil)
if err != nil {
t.Fatalf("planEbookSeriesWrite: %v", err)
}
if plan.Kind != ebookSeriesWriteNone {
t.Fatalf("identical plan = %+v, want none", plan)
}
}
func TestPlanEbookSeriesWriteAllowsNullNumericIndex(t *testing.T) {
plan, err := planEbookSeriesWrite(&parsedEbook{Series: "Series", SeriesIndex: "appendix"}, nil, nil, pgx.ErrNoRows)
if err != nil {
t.Fatalf("planEbookSeriesWrite: %v", err)
}
if plan.Kind != ebookSeriesWriteUpsert || plan.Index != nil {
t.Fatalf("plan = %+v, want upsert with nil index", plan)
}
}
func TestPlanEbookSeriesWriteReturnsQueryError(t *testing.T) {
queryErr := errors.New("query failed")
if _, err := planEbookSeriesWrite(&parsedEbook{Series: "Series"}, nil, nil, queryErr); !errors.Is(err, queryErr) {
t.Fatalf("planEbookSeriesWrite error = %v, want query error", err)
}
}
func TestEbookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
book := &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
if got := ebookIdentityConfidence(book); got != "high" {
t.Fatalf("complete metadata confidence = %q, want high", got)
}
book = &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024}
if got := ebookIdentityConfidence(book); got != "medium" {
t.Fatalf("partial metadata confidence = %q, want medium", got)
}
book = &parsedEbook{}
if got := ebookIdentityConfidence(book); got != "low" {
t.Fatalf("empty metadata confidence = %q, want low", got)
}
}
func TestResolveEbookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
finder := &fakeRootContentFinder{}
writer := &fakeFilesystemItemWriter{}
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{
Title: "Book", Year: 2024, Authors: []string{"Author"}, Description: "Overview", Publisher: "Publisher", Genres: []string{"Fiction"}, Language: "en",
})
if err != nil {
t.Fatalf("resolveEbookMediaItem: %v", err)
}
if got == "" || len(writer.upserts) != 1 {
t.Fatalf("contentID/upserts = %q/%d, want id and one upsert", got, len(writer.upserts))
}
item := writer.upserts[0]
if item.Type != "ebook" || item.Title != "Book" || item.Year != 2024 || item.Overview != "Overview" || item.OriginalLanguage != "en" {
t.Fatalf("upserted ebook item = %+v", item)
}
}
func TestResolveEbookMediaItemReusesRootScopedContentID(t *testing.T) {
finder := &fakeRootContentFinder{contentID: "ebook-root-id"}
writer := &fakeFilesystemItemWriter{}
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{Title: "Book"})
if err != nil {
t.Fatalf("resolveEbookMediaItem: %v", err)
}
if got != "ebook-root-id" {
t.Fatalf("contentID = %q, want root-scoped id", got)
}
if len(writer.upserts) != 0 {
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
}
}
func TestResolveEbookExistingRootAppliesParsedMetadata(t *testing.T) {
reader := &fakeFilesystemItemReader{items: []*models.MediaItem{{
ContentID: "ebook-root-id",
Type: "ebook",
Title: "Old Title",
Year: 2020,
OriginalLanguage: "",
}}}
writer := &fakeFilesystemItemWriter{}
err := updateExistingEbookMediaItem(context.Background(), reader, writer, "ebook-root-id", &parsedEbook{
Title: "New Title",
Year: 2026,
Publisher: "New Press",
Genres: []string{"Fiction"},
Language: "en",
})
if err != nil {
t.Fatalf("updateExistingEbookMediaItem: %v", err)
}
if len(writer.upserts) != 1 {
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
}
item := writer.upserts[0]
if item.ContentID != "ebook-root-id" || item.Type != "ebook" || item.Title != "New Title" || item.Year != 2026 || item.OriginalLanguage != "en" {
t.Fatalf("updated item = %+v", item)
}
if strings.Join(item.Studios, ",") != "New Press" || strings.Join(item.Genres, ",") != "Fiction" {
t.Fatalf("updated item studios/genres = %+v/%+v", item.Studios, item.Genres)
}
}
func TestApplyEbookToMediaItemPreservesExistingYearWhenParsedYearMissing(t *testing.T) {
item := &models.MediaItem{
Type: "ebook",
Title: "Existing",
Year: 1999,
}
applyEbookToMediaItem(item, &parsedEbook{Title: "Updated", Year: 0})
if item.Title != "Updated" {
t.Fatalf("Title = %q, want Updated", item.Title)
}
if item.Year != 1999 {
t.Fatalf("Year = %d, want preserved 1999", item.Year)
}
}
func TestEbookPeopleCreditsEqualAuthorsOnly(t *testing.T) {
existing := []models.ItemPerson{
{Person: models.Person{Name: "Ada Writer"}, Kind: models.PersonKindAuthor, SortOrder: 0},
{Person: models.Person{Name: "Ben Author"}, Kind: models.PersonKindAuthor, SortOrder: 1},
{Person: models.Person{Name: "Manual Contributor"}, Kind: models.PersonKindWriter, SortOrder: 2},
}
desired := []ebookCredit{
{Name: "Ada Writer", Kind: models.PersonKindAuthor},
{Name: "Ben Author", Kind: models.PersonKindAuthor},
}
if !ebookPeopleCreditsEqual(existing, desired) {
t.Fatal("expected matching author credits to compare equal while ignoring non-authors")
}
existing[1] = models.ItemPerson{Person: models.Person{Name: "Other Author"}, Kind: models.PersonKindAuthor, SortOrder: 1}
if ebookPeopleCreditsEqual(existing, desired) {
t.Fatal("expected different author credit to make ebook author set differ")
}
}
func TestEbookPeopleMergePreservesExistingNonAuthorCredits(t *testing.T) {
existing := []models.ItemPerson{
{Person: models.Person{ID: 10, Name: "Old Author"}, Kind: models.PersonKindAuthor, SortOrder: 0},
{Person: models.Person{ID: 20, Name: "Manual Writer"}, Kind: models.PersonKindWriter, SortOrder: 1, Character: "essay"},
{Person: models.Person{ID: 30, Name: "Manual Narrator"}, Kind: models.PersonKindNarrator, SortOrder: 2},
}
authors := []ebookResolvedAuthor{
{ID: 40, Name: "New Author"},
}
got := mergeEbookPeople(existing, authors)
if len(got) != 3 {
t.Fatalf("merged people len = %d, want 3: %+v", len(got), got)
}
if got[0].Person.ID != 20 || got[0].Kind != models.PersonKindWriter || got[0].Character != "essay" {
t.Fatalf("first preserved non-author = %+v", got[0])
}
if got[1].Person.ID != 30 || got[1].Kind != models.PersonKindNarrator {
t.Fatalf("second preserved non-author = %+v", got[1])
}
if got[2].Person.ID != 40 || got[2].Kind != models.PersonKindAuthor || got[2].SortOrder != 2 {
t.Fatalf("new author credit = %+v", got[2])
}
}
func TestEbookPeopleReplacePlanReturnsGetPeopleError(t *testing.T) {
wantErr := errors.New("get people failed")
_, err := ebookPeopleForReplace(nil, wantErr, []ebookResolvedAuthor{{ID: 40, Name: "New Author"}})
if !errors.Is(err, wantErr) {
t.Fatalf("error = %v, want %v", err, wantErr)
}
}
func TestScanEbookBuildMediaFileSetsCorePersistenceFields(t *testing.T) {
modifiedAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
book := &parsedEbook{Format: "epub", Title: "Book", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
got := buildEbookMediaFile(&models.MediaFolder{ID: 44}, "content-1", "/library/Book.epub", 1234, modifiedAt, book)
if got.ContentID != "content-1" || got.MediaFolderID != 44 {
t.Fatalf("ids = %q/%d, want content-1/44", got.ContentID, got.MediaFolderID)
}
if got.BaseType != "ebook" || got.BaseTitle != "Book" || got.BaseYear != 2024 || got.Container != "epub" || got.ProbeSource != "local" {
t.Fatalf("ebook file metadata = %+v", got)
}
if got.CanonicalRootPath != "/library/Book.epub" || got.ObservedRootPath != "/library/Book.epub" || got.FilePath != "/library/Book.epub" {
t.Fatalf("ebook paths = canonical %q observed %q file %q", got.CanonicalRootPath, got.ObservedRootPath, got.FilePath)
}
if got.FileSize != 1234 || got.FileModifiedAt == nil || !got.FileModifiedAt.Equal(modifiedAt) {
t.Fatalf("file facts = size %d modified %v", got.FileSize, got.FileModifiedAt)
}
}
func TestScanEbookPersistenceSQLWritesLibraryMembershipAndISBNOnly(t *testing.T) {
ctx := context.Background()
exec := &recordingEbookExecutor{}
if err := insertEbookLibraryMembership(ctx, exec, "content-1", 44); err != nil {
t.Fatalf("insertEbookLibraryMembership: %v", err)
}
if err := insertEbookISBNProviderID(ctx, exec, "content-1", "9780306406157"); err != nil {
t.Fatalf("insertEbookISBNProviderID: %v", err)
}
if len(exec.queries) != 2 {
t.Fatalf("queries = %d, want 2", len(exec.queries))
}
if !strings.Contains(exec.queries[0], "media_item_libraries") || exec.args[0][0] != "content-1" || exec.args[0][1] != 44 {
t.Fatalf("library membership write = query %q args %+v", exec.queries[0], exec.args[0])
}
if !strings.Contains(exec.queries[1], "media_item_provider_ids") || !strings.Contains(exec.queries[1], "'isbn'") || !strings.Contains(exec.queries[1], "'ebook'") {
t.Fatalf("ISBN provider write query = %q", exec.queries[1])
}
if strings.Contains(exec.queries[1], "asin") || exec.args[1][0] != "content-1" || exec.args[1][1] != "9780306406157" {
t.Fatalf("ISBN provider write args/query = query %q args %+v", exec.queries[1], exec.args[1])
}
}
func TestScanEbookFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "bad.epub"), []byte("not a real epub"), 0o644); err != nil {
t.Fatalf("write fake ebook: %v", err)
}
s := &Scanner{}
err := s.ScanEbookFolder(context.Background(), &models.MediaFolder{ID: 44, Paths: []string{root}})
if err == nil {
t.Fatal("ScanEbookFolder returned nil, want aggregate failure")
}
if !strings.Contains(err.Error(), "folder_id=44") {
t.Fatalf("error = %q, want folder id", err)
}
}
func TestScanEbookFolderReturnsCanceledContext(t *testing.T) {
root := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := &Scanner{}
err := s.ScanEbookFolder(ctx, &models.MediaFolder{ID: 44, Paths: []string{root}})
if !errors.Is(err, context.Canceled) {
t.Fatalf("ScanEbookFolder error = %v, want context.Canceled", err)
}
}
func writeTestEPUB(t *testing.T, identifiers []string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "book.epub")
file, err := os.Create(path)
if err != nil {
t.Fatalf("create epub: %v", err)
}
zw := zip.NewWriter(file)
add := func(name, body string) {
t.Helper()
w, err := zw.Create(name)
if err != nil {
t.Fatalf("create zip entry %s: %v", name, err)
}
if _, err := w.Write([]byte(body)); err != nil {
t.Fatalf("write zip entry %s: %v", name, err)
}
}
add("META-INF/container.xml", `<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>`)
var identifierXML strings.Builder
for _, identifier := range identifiers {
identifierXML.WriteString(" <dc:identifier>")
identifierXML.WriteString(identifier)
identifierXML.WriteString("</dc:identifier>\n")
}
add("OPS/content.opf", `<?xml version="1.0" encoding="UTF-8"?>
<package xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
<metadata>
<dc:title>The Test Ebook</dc:title>
<dc:creator>Ada Writer</dc:creator>
<dc:creator>Ben Author</dc:creator>
`+identifierXML.String()+`
<dc:publisher>Silo Press</dc:publisher>
<dc:date>2024-03-10</dc:date>
<dc:language>en</dc:language>
<dc:subject>Fiction</dc:subject>
<dc:subject>Adventure</dc:subject>
<dc:description>Back cover copy</dc:description>
<meta name="calibre:series" content="Test Series"/>
<meta name="calibre:series_index" content="2"/>
</metadata>
</package>`)
if err := zw.Close(); err != nil {
t.Fatalf("close zip: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("close epub: %v", err)
}
return path
}
+26
View File
@@ -241,15 +241,27 @@ func (s *Scanner) ScanFolder(ctx context.Context, folder *models.MediaFolder) (*
return &ScanResult{}, nil
}
if isEbookLibraryType(folder.Type) {
if err := s.ScanEbookFolder(watchCtx, folder); err != nil {
return nil, err
}
return &ScanResult{}, nil
}
return s.scanPaths(watchCtx, folder, folder.Paths, folder.Paths, true)
}
var errEbookScanningNotImplemented = errors.New("ebook scanning is not implemented")
// ScanSubtree walks a single subtree within a media folder and reconciles only
// files that live beneath that subtree.
func (s *Scanner) ScanSubtree(ctx context.Context, folder *models.MediaFolder, subtreePath string) (*ScanResult, error) {
cleanSubtree := filepath.Clean(subtreePath)
watchCtx, stopWatch := s.watchFolderContext(ctx, folder.ID)
defer stopWatch()
if isEbookLibraryType(folder.Type) {
return nil, errEbookScanningNotImplemented
}
return s.scanPaths(watchCtx, folder, []string{cleanSubtree}, []string{cleanSubtree}, false)
}
@@ -278,6 +290,15 @@ func isPodcastLibraryType(libraryType string) bool {
}
}
func isEbookLibraryType(libraryType string) bool {
switch strings.ToLower(strings.TrimSpace(libraryType)) {
case "ebook", "ebooks":
return true
default:
return false
}
}
// walkMode tells walkLogicalTree which file extensions to surface and
// which library-specific filename heuristics (sample/extra skipping)
// to apply.
@@ -288,6 +309,7 @@ const (
walkModeMovie // movie library: video extensions + sample/extra skipping
walkModeAudiobook // audiobook library: audio extensions, no skipping
walkModePodcast // podcast library: audio extensions, no skipping
walkModeEbook // ebook library: ebook extensions, no skipping
)
// walkModeFor derives a walkMode from a media_folders.type string.
@@ -301,6 +323,8 @@ func walkModeFor(folderType string) walkMode {
return walkModeAudiobook
case isPodcastLibraryType(folderType):
return walkModePodcast
case isEbookLibraryType(folderType):
return walkModeEbook
default:
return walkModeVideo
}
@@ -312,6 +336,8 @@ func (m walkMode) acceptsExt(ext string) bool {
switch m {
case walkModeAudiobook, walkModePodcast:
return audioExtensions[ext]
case walkModeEbook:
return ebookExtensions[ext]
default:
return videoExtensions[ext]
}
+80
View File
@@ -2,6 +2,7 @@ package scanner
import (
"context"
"errors"
"os"
"path/filepath"
"sort"
@@ -200,3 +201,82 @@ func TestIsPodcastLibraryType(t *testing.T) {
}
}
}
func TestIsEbookLibraryType(t *testing.T) {
cases := []struct {
in string
want bool
}{
{"ebooks", true},
{"ebook", true},
{"Ebook", true},
{" EBOOKS ", true},
{"audiobooks", false},
{"movies", false},
{"", false},
}
for _, tc := range cases {
if got := isEbookLibraryType(tc.in); got != tc.want {
t.Errorf("isEbookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
}
}
}
func TestWalkModeForEbookLibraryTypes(t *testing.T) {
for _, libraryType := range []string{"ebook", "ebooks", " EBOOKS "} {
if got := walkModeFor(libraryType); got != walkModeEbook {
t.Fatalf("walkModeFor(%q) = %v, want %v", libraryType, got, walkModeEbook)
}
}
}
func TestWalkModeEbookAcceptsEbookExtensionsOnly(t *testing.T) {
if !walkModeEbook.acceptsExt(".epub") {
t.Fatal("walkModeEbook should accept .epub")
}
if !walkModeEbook.acceptsExt(".pdf") {
t.Fatal("walkModeEbook should accept .pdf")
}
if walkModeEbook.acceptsExt(".mp4") {
t.Fatal("walkModeEbook should reject video extensions")
}
if walkModeEbook.acceptsExt(".mp3") {
t.Fatal("walkModeEbook should reject audio extensions")
}
}
func TestScanFolderEbookLibraryRoutesToEbookScanner(t *testing.T) {
scanner := &Scanner{}
result, err := scanner.ScanFolder(context.Background(), &models.MediaFolder{
ID: 0,
Type: " ebooks ",
Paths: []string{t.TempDir()},
})
if err != nil {
t.Fatalf("ScanFolder ebook error = %v, want nil", err)
}
if result == nil {
t.Fatal("ScanFolder ebook result = nil, want empty result")
}
}
func TestScanSubtreeEbookLibraryReturnsNotImplemented(t *testing.T) {
subtree := t.TempDir()
ebookPath := filepath.Join(subtree, "Book.epub")
if err := os.WriteFile(ebookPath, []byte("not a real epub"), 0o644); err != nil {
t.Fatalf("write ebook fixture: %v", err)
}
scanner := &Scanner{}
result, err := scanner.ScanSubtree(context.Background(), &models.MediaFolder{
ID: 0,
Type: "ebook",
Paths: []string{subtree},
}, subtree)
if !errors.Is(err, errEbookScanningNotImplemented) {
t.Fatalf("ScanSubtree ebook error = %v, want %v", err, errEbookScanningNotImplemented)
}
if result != nil {
t.Fatalf("ScanSubtree ebook result = %+v, want nil", result)
}
}
@@ -0,0 +1,56 @@
package tasks
import (
"context"
"encoding/json"
"fmt"
"github.com/Silo-Server/silo-server/internal/taskmanager"
)
type ebookMetadataEnricher interface {
Run(ctx context.Context) (int, error)
}
// SyncEbookMetadataTask runs the periodic ebook enrichment sweep.
// It calls ebooks.Enricher.Run() which selects unenriched ebook media_items,
// resolves the per-folder metadata-provider chain at content_level='ebook',
// and writes results back to the database.
type SyncEbookMetadataTask struct {
enricher ebookMetadataEnricher
}
// NewSyncEbookMetadataTask constructs the task.
func NewSyncEbookMetadataTask(enricher ebookMetadataEnricher) *SyncEbookMetadataTask {
return &SyncEbookMetadataTask{enricher: enricher}
}
func (t *SyncEbookMetadataTask) Key() string { return "sync_ebook_metadata" }
func (t *SyncEbookMetadataTask) Name() string { return "Sync Ebook Metadata" }
func (t *SyncEbookMetadataTask) Description() string {
return "Fetches metadata (cover art, overview, authors) for ebooks that have not yet been enriched"
}
func (t *SyncEbookMetadataTask) Category() taskmanager.TaskCategory {
return taskmanager.TaskCategoryMetadata
}
func (t *SyncEbookMetadataTask) IsHidden() bool { return false }
func (t *SyncEbookMetadataTask) DefaultTriggers() []taskmanager.TriggerConfig {
return []taskmanager.TriggerConfig{
{Type: taskmanager.TriggerTypeInterval, IntervalMs: 5 * 60 * 1000},
}
}
func (t *SyncEbookMetadataTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error {
progress.Report(0, "Scanning for unenriched ebooks")
enriched, err := t.enricher.Run(ctx)
if err != nil {
return fmt.Errorf("ebook metadata sync: %w", err)
}
result, _ := json.Marshal(map[string]int{"items_enriched": enriched})
progress.SetResultData(result)
progress.Report(100, fmt.Sprintf("Ebook metadata sync complete (%d items enriched)", enriched))
return nil
}
@@ -0,0 +1,92 @@
package tasks
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/Silo-Server/silo-server/internal/taskmanager"
)
type fakeEbookMetadataEnricher struct {
enriched int
err error
called bool
}
func (f *fakeEbookMetadataEnricher) Run(context.Context) (int, error) {
f.called = true
return f.enriched, f.err
}
type ebookMetadataProgressReporter struct {
messages []string
result json.RawMessage
}
func (p *ebookMetadataProgressReporter) Report(_ float64, message string) {
p.messages = append(p.messages, message)
}
func (p *ebookMetadataProgressReporter) SetResultData(data json.RawMessage) {
p.result = data
}
func TestSyncEbookMetadataTaskProperties(t *testing.T) {
task := NewSyncEbookMetadataTask(&fakeEbookMetadataEnricher{})
if task.Key() != "sync_ebook_metadata" {
t.Fatalf("Key() = %q, want sync_ebook_metadata", task.Key())
}
if task.Name() != "Sync Ebook Metadata" {
t.Fatalf("Name() = %q, want Sync Ebook Metadata", task.Name())
}
if task.Category() != taskmanager.TaskCategoryMetadata {
t.Fatalf("Category() = %q, want %q", task.Category(), taskmanager.TaskCategoryMetadata)
}
if task.IsHidden() {
t.Fatal("IsHidden() = true, want false")
}
triggers := task.DefaultTriggers()
if len(triggers) != 1 || triggers[0].Type != taskmanager.TriggerTypeInterval || triggers[0].IntervalMs != 5*60*1000 {
t.Fatalf("DefaultTriggers() = %#v, want one 5 minute interval", triggers)
}
if strings.Contains(strings.ToLower(task.Description()), "narrator") {
t.Fatalf("Description() mentions narrator: %q", task.Description())
}
}
func TestSyncEbookMetadataTaskExecuteReportsResult(t *testing.T) {
enricher := &fakeEbookMetadataEnricher{enriched: 3}
task := NewSyncEbookMetadataTask(enricher)
progress := &ebookMetadataProgressReporter{}
if err := task.Execute(context.Background(), progress); err != nil {
t.Fatalf("Execute() error = %v", err)
}
if !enricher.called {
t.Fatal("enricher was not called")
}
var result map[string]int
if err := json.Unmarshal(progress.result, &result); err != nil {
t.Fatalf("result data JSON error: %v", err)
}
if result["items_enriched"] != 3 {
t.Fatalf("items_enriched = %d, want 3", result["items_enriched"])
}
if len(progress.messages) == 0 || !strings.Contains(progress.messages[len(progress.messages)-1], "3 items enriched") {
t.Fatalf("progress messages = %#v, want completion count", progress.messages)
}
}
func TestSyncEbookMetadataTaskExecuteWrapsError(t *testing.T) {
enricher := &fakeEbookMetadataEnricher{err: errors.New("boom")}
task := NewSyncEbookMetadataTask(enricher)
err := task.Execute(context.Background(), &ebookMetadataProgressReporter{})
if err == nil || !strings.Contains(err.Error(), "ebook metadata sync") {
t.Fatalf("Execute() error = %v, want wrapped ebook metadata sync error", err)
}
}
@@ -0,0 +1,17 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE ebook_series (
content_id TEXT PRIMARY KEY REFERENCES media_items(content_id) ON DELETE CASCADE,
series_name TEXT NOT NULL,
series_index NUMERIC,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX ebook_series_name_lower
ON ebook_series (LOWER(series_name), series_index NULLS LAST);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS ebook_series;
-- +goose StatementEnd
@@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import { contentLevelsForType } from "./LibraryForm";
describe("contentLevelsForType", () => {
it("maps ebook libraries to the ebook metadata content level", () => {
expect(contentLevelsForType("ebooks")).toEqual(["ebook"]);
expect(contentLevelsForType("ebook")).toEqual(["ebook"]);
});
it("includes ebook metadata providers for mixed libraries", () => {
expect(contentLevelsForType("mixed")).toEqual([
"movie",
"series",
"season",
"episode",
"audiobook",
"ebook",
]);
});
});
@@ -59,16 +59,19 @@ export interface LibraryFormProps {
extraContent?: ReactNode;
}
function contentLevelsForType(libraryType: string): string[] {
export function contentLevelsForType(libraryType: string): string[] {
switch (libraryType) {
case "series":
return ["series", "season", "episode"];
case "movies":
return ["movie"];
case "mixed":
return ["movie", "series", "season", "episode"];
return ["movie", "series", "season", "episode", "audiobook", "ebook"];
case "audiobooks":
return ["audiobook"];
case "ebooks":
case "ebook":
return ["ebook"];
case "podcasts":
return ["podcast", "podcast_episode"];
default:
@@ -474,6 +477,7 @@ export function LibraryForm({
<SelectItem value="series">Series</SelectItem>
<SelectItem value="mixed">Mixed</SelectItem>
<SelectItem value="audiobooks">Audiobooks</SelectItem>
<SelectItem value="ebooks">Ebooks</SelectItem>
<SelectItem value="podcasts">Podcasts</SelectItem>
</SelectContent>
</Select>