Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e7189f074 | ||
|
|
21170cfcf3 | ||
|
|
0e1c2995a3 | ||
|
|
98d213204a | ||
|
|
5ea8868047 | ||
|
|
0b97d2bd6d | ||
|
|
293db1f255 | ||
|
|
1119cef018 | ||
|
|
c91a1d5b29 | ||
|
|
f15423d505 | ||
|
|
6de2b19c07 | ||
|
|
adf595cbf9 | ||
|
|
0b44952816 |
@@ -0,0 +1,466 @@
|
||||
# Ebooks Like Audiobooks Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add ebooks as a core silo media type by mirroring the audiobook scanner architecture in `main`. Metadata enrichment is documented below as follow-up PR work, not part of the foundation PR.
|
||||
|
||||
**Architecture:** The foundation PR scans ebooks through core `internal/scanner` code. Later PRs add a core `internal/ebooks.Enricher` task that resolves plugin metadata providers with `content_level = 'ebook'`. The first-party `silo-plugin-ebook-metadata` provides lookup data; there is no ebook scanner plugin and no ebook details repository/table.
|
||||
|
||||
**Tech Stack:** Go, PostgreSQL Goose migrations, Silo scanner/catalog repositories, Silo metadata plugin chain, taskmanager scheduled tasks.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- `internal/scanner/ebook.go`: local ebook parser and format support.
|
||||
- `internal/scanner/ebook_scan.go`: ebook scan flow matching `audiobook_scan.go`.
|
||||
- `internal/scanner/scanner.go`: route `ebook(s)` libraries to the ebook scanner and add ebook walk mode.
|
||||
- `migrations/sql/20260608000100_ebook_series.sql`: `ebook_series` table equivalent to `audiobook_series`.
|
||||
- Follow-up PR: `internal/ebooks/enrichment.go`: ebook metadata sweep equivalent to `internal/audiobooks/enrichment.go`.
|
||||
- Follow-up PR: `internal/taskmanager/tasks/sync_ebook_metadata.go`: scheduled/manual metadata task equivalent to `sync_audiobook_metadata.go`.
|
||||
- Follow-up PR: `cmd/silo/main.go`: wire `ebooks.NewEnricher` next to `audiobooks.NewEnricher`.
|
||||
- Tests live beside changed files and should pin “authors only, ISBN only, content_level ebook.”
|
||||
|
||||
### Task 1: Scanner Routing And Local Parser
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/scanner/scanner.go`
|
||||
- Modify: `internal/scanner/scanner_test.go`
|
||||
- Create/modify: `internal/scanner/ebook.go`
|
||||
- Create/modify: `internal/scanner/ebook_test.go`
|
||||
|
||||
- [ ] **Step 1: Write or keep failing scanner/parser tests**
|
||||
|
||||
Ensure these tests exist:
|
||||
|
||||
```go
|
||||
func TestIsEbookLibraryType(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"ebooks", true},
|
||||
{"ebook", true},
|
||||
{"Ebook", true},
|
||||
{" EBOOKS ", true},
|
||||
{"audiobooks", false},
|
||||
{"movies", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := isEbookLibraryType(tc.in); got != tc.want {
|
||||
t.Errorf("isEbookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkModeForEbookLibraryTypes(t *testing.T) {
|
||||
for _, libraryType := range []string{"ebook", "ebooks", " EBOOKS "} {
|
||||
if got := walkModeFor(libraryType); got != walkModeEbook {
|
||||
t.Fatalf("walkModeFor(%q) = %v, want walkModeEbook", libraryType, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsEbookFile(t *testing.T) {
|
||||
cases := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"book.epub", true},
|
||||
{"book.PDF", true},
|
||||
{"book.azw3", true},
|
||||
{"book.mp3", false},
|
||||
{"movie.mkv", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := SupportsEbookFile(tc.path); got != tc.want {
|
||||
t.Errorf("SupportsEbookFile(%q) = %v, want %v", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run scanner/parser red test**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -run 'Test(IsEbook|WalkModeForEbook|WalkModeEbook|SupportsEbook|ParseEbook)' -count=1
|
||||
```
|
||||
|
||||
Expected before implementation: fail for missing ebook mode/parser functions. Expected after existing parser work is kept: pass.
|
||||
|
||||
- [ ] **Step 3: Implement minimal parser and walk mode**
|
||||
|
||||
Implement:
|
||||
|
||||
```go
|
||||
var ebookExtensions = map[string]bool{
|
||||
".epub": true,
|
||||
".pdf": true,
|
||||
".mobi": true,
|
||||
".azw": true,
|
||||
".azw3": true,
|
||||
".fb2": true,
|
||||
}
|
||||
|
||||
func SupportsEbookFile(filePath string) bool {
|
||||
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
|
||||
}
|
||||
|
||||
func isEbookLibraryType(libraryType string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(libraryType)) {
|
||||
case "ebook", "ebooks":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Add `walkModeEbook` and make `walkMode.acceptsExt` call `SupportsEbookFile`.
|
||||
|
||||
- [ ] **Step 4: Run scanner/parser green test**
|
||||
|
||||
Run the same command as Step 2. Expected: `ok github.com/Silo-Server/silo-server/internal/scanner`.
|
||||
|
||||
- [ ] **Step 5: Commit Task 1**
|
||||
|
||||
```bash
|
||||
git add internal/scanner/scanner.go internal/scanner/scanner_test.go internal/scanner/ebook.go internal/scanner/ebook_test.go
|
||||
git commit -m "feat: add ebook scanner parser foundation"
|
||||
```
|
||||
|
||||
### Task 2: Ebook Core Scan Persistence
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/scanner/ebook_scan.go`
|
||||
- Modify: `internal/scanner/ebook_test.go`
|
||||
- Modify: `internal/scanner/scanner.go`
|
||||
|
||||
- [ ] **Step 1: Write failing tests for audiobook-shaped ebook helpers**
|
||||
|
||||
Add tests for:
|
||||
|
||||
```go
|
||||
func TestEbookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
|
||||
book := &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
|
||||
if got := ebookIdentityConfidence(book); got != "high" {
|
||||
t.Fatalf("complete metadata confidence = %q, want high", got)
|
||||
}
|
||||
book = &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024}
|
||||
if got := ebookIdentityConfidence(book); got != "medium" {
|
||||
t.Fatalf("partial metadata confidence = %q, want medium", got)
|
||||
}
|
||||
book = &parsedEbook{}
|
||||
if got := ebookIdentityConfidence(book); got != "low" {
|
||||
t.Fatalf("empty metadata confidence = %q, want low", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEbookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{
|
||||
Title: "Book", Year: 2024, Authors: []string{"Author"}, Description: "Overview", Publisher: "Publisher", Genres: []string{"Fiction"}, Language: "en",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveEbookMediaItem: %v", err)
|
||||
}
|
||||
if got == "" || len(writer.upserts) != 1 {
|
||||
t.Fatalf("contentID/upserts = %q/%d, want id and one upsert", got, len(writer.upserts))
|
||||
}
|
||||
item := writer.upserts[0]
|
||||
if item.Type != "ebook" || item.Title != "Book" || item.Year != 2024 || item.Overview != "Overview" || item.OriginalLanguage != "en" {
|
||||
t.Fatalf("upserted ebook item = %+v", item)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run red helper tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -run 'Test(EbookIdentity|ResolveEbook|EbookPeople)' -count=1
|
||||
```
|
||||
|
||||
Expected before implementation: fail with undefined ebook scan helper functions.
|
||||
|
||||
- [ ] **Step 3: Implement ebook scan helpers mirroring audiobook names**
|
||||
|
||||
Create `internal/scanner/ebook_scan.go` with functions shaped after `audiobook_scan.go`:
|
||||
|
||||
```go
|
||||
func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolder) error
|
||||
func (s *Scanner) reconcileEbookFile(ctx context.Context, folder *models.MediaFolder, filePath string, skipped *int64) error
|
||||
func (s *Scanner) ebookFileShouldSkip(ctx context.Context, folder *models.MediaFolder, filePath string, size int64, modifiedAt time.Time) (bool, error)
|
||||
func (s *Scanner) upsertEbookMediaItem(ctx context.Context, folderID int, filePath string, book *parsedEbook) (string, error)
|
||||
func resolveEbookMediaItem(ctx context.Context, rootFinder filesystemRootContentFinder, itemWriter filesystemMediaItemWriter, folderID int, filePath string, book *parsedEbook) (string, error)
|
||||
func createEbookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedEbook, cleanTitle string) (string, error)
|
||||
func applyEbookToMediaItem(item *models.MediaItem, book *parsedEbook)
|
||||
func (s *Scanner) upsertEbookMediaFile(ctx context.Context, folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) error
|
||||
func (s *Scanner) upsertEbookPeople(ctx context.Context, contentID string, book *parsedEbook) error
|
||||
func (s *Scanner) upsertEbookSeries(ctx context.Context, contentID string, book *parsedEbook) error
|
||||
func ebookIdentityConfidence(book *parsedEbook) string
|
||||
```
|
||||
|
||||
Behavior:
|
||||
- `ScanEbookFolder` walks `folder.Paths`, collects files where `SupportsEbookFile` is true, processes in a worker pool like `ScanAudiobookFolder`, logs `ebook scan: starting/progress/completed/indexed`.
|
||||
- `upsertEbookMediaItem` reuses `FindContentIDByRootPath(ctx, folderID, filePath, "ebook")`, then finds by existing `media_files.file_path`, then creates a new `media_items` row.
|
||||
- `applyEbookToMediaItem` sets type/title/year/overview/publisher-as-studio/genres/release date/original language.
|
||||
- `upsertEbookMediaFile` writes one `media_files` row with `CanonicalRootPath`, `ObservedRootPath`, `ContentGroupKey`, `FilePath` all tied to the ebook file path or content ID in the same style as audiobook file persistence.
|
||||
- `upsertEbookPeople` writes author credits only with `models.PersonKindAuthor`; it never writes `models.PersonKindNarrator`.
|
||||
- `reconcileEbookFile` writes ISBN provider ID with:
|
||||
|
||||
```sql
|
||||
INSERT INTO media_item_provider_ids (content_id, provider, provider_id, item_type)
|
||||
VALUES ($1, 'isbn', $2, 'ebook')
|
||||
ON CONFLICT DO NOTHING
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Route ebook libraries in `ScanFolder`**
|
||||
|
||||
Add this block beside audiobook/podcast routing:
|
||||
|
||||
```go
|
||||
if isEbookLibraryType(folder.Type) {
|
||||
if err := s.ScanEbookFolder(watchCtx, folder); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ScanResult{}, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run green helper tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -run 'Test(EbookIdentity|ResolveEbook|EbookPeople|ScanEbook)' -count=1
|
||||
```
|
||||
|
||||
Expected: scanner tests pass.
|
||||
|
||||
- [ ] **Step 6: Commit Task 2**
|
||||
|
||||
```bash
|
||||
git add internal/scanner/scanner.go internal/scanner/ebook_scan.go internal/scanner/ebook_test.go
|
||||
git commit -m "feat: scan ebook libraries in core"
|
||||
```
|
||||
|
||||
### Task 3: Ebook Series Migration
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/sql/20260608000100_ebook_series.sql`
|
||||
- Modify: `internal/scanner/ebook_scan.go`
|
||||
|
||||
- [ ] **Step 1: Add migration mirroring `audiobook_series`**
|
||||
|
||||
Create:
|
||||
|
||||
```sql
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE ebook_series (
|
||||
content_id TEXT PRIMARY KEY REFERENCES media_items(content_id) ON DELETE CASCADE,
|
||||
series_name TEXT NOT NULL,
|
||||
series_index NUMERIC,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX ebook_series_name_lower
|
||||
ON ebook_series (LOWER(series_name), series_index NULLS LAST);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS ebook_series;
|
||||
-- +goose StatementEnd
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add scanner series upsert SQL**
|
||||
|
||||
Use the same write/delete/skip-identical shape as `upsertAudiobookSeries`, with table name `ebook_series` and `book.SeriesIndex` parsed through existing `parseSeriesIndex`.
|
||||
|
||||
- [ ] **Step 3: Run migration and scanner tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner -run 'Test(Ebook|ResolveEbook)' -count=1
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
- [ ] **Step 4: Commit Task 3**
|
||||
|
||||
```bash
|
||||
git add migrations/sql/20260608000100_ebook_series.sql internal/scanner/ebook_scan.go
|
||||
git commit -m "feat: persist ebook series membership"
|
||||
```
|
||||
|
||||
### Follow-up PR Task 4: Ebook Metadata Enricher
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/ebooks/enrichment.go`
|
||||
- Create: `internal/ebooks/enrichment_test.go`
|
||||
|
||||
- [ ] **Step 1: Copy audiobook enricher structure with ebook domain changes**
|
||||
|
||||
Create `internal/ebooks/enrichment.go` from `internal/audiobooks/enrichment.go` and change:
|
||||
- package `ebooks`
|
||||
- log prefix `ebook enrichment`
|
||||
- default env var `SILO_EBOOK_ENRICH_WORKERS`
|
||||
- batch query `WHERE mi.type = 'ebook'`
|
||||
- chain resolution `metadata.ResolveChain(ctx, item.FolderID, "ebook", e.chainRepo, e.resolver)`
|
||||
- search query `ContentType: "ebook"`
|
||||
- metadata request `ContentType: "ebook"`
|
||||
- people persistence filters to author credits only; discard narrator credits if a plugin returns them
|
||||
- no local ffmpeg audiobook cover fallback
|
||||
|
||||
- [ ] **Step 2: Add tests for chain content level and authors-only people**
|
||||
|
||||
Add tests that instantiate an `Enricher` with fake provider behavior where possible, or unit-test narrow helpers:
|
||||
|
||||
```go
|
||||
func TestEbookEnricherContentType(t *testing.T) {
|
||||
if got := ebookContentType(); got != "ebook" {
|
||||
t.Fatalf("ebookContentType() = %q, want ebook", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterEbookPeopleKeepsAuthorsOnly(t *testing.T) {
|
||||
in := []models.ItemPerson{
|
||||
{Person: models.Person{Name: "Author"}, Kind: models.PersonKindAuthor},
|
||||
{Person: models.Person{Name: "Narrator"}, Kind: models.PersonKindNarrator},
|
||||
}
|
||||
got := filterEbookPeople(in)
|
||||
if len(got) != 1 || got[0].Kind != models.PersonKindAuthor || got[0].Person.Name != "Author" {
|
||||
t.Fatalf("filterEbookPeople() = %+v, want author only", got)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run red/green ebook enricher tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/ebooks -count=1
|
||||
```
|
||||
|
||||
Expected after implementation: pass.
|
||||
|
||||
- [ ] **Step 4: Commit Task 4**
|
||||
|
||||
```bash
|
||||
git add internal/ebooks/enrichment.go internal/ebooks/enrichment_test.go
|
||||
git commit -m "feat: add ebook metadata enricher"
|
||||
```
|
||||
|
||||
### Follow-up PR Task 5: Ebook Metadata Task And Main Wiring
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/taskmanager/tasks/sync_ebook_metadata.go`
|
||||
- Modify: `cmd/silo/main.go`
|
||||
|
||||
- [ ] **Step 1: Add sync task matching audiobook task**
|
||||
|
||||
Create task:
|
||||
|
||||
```go
|
||||
type SyncEbookMetadataTask struct {
|
||||
enricher *ebooks.Enricher
|
||||
}
|
||||
|
||||
func NewSyncEbookMetadataTask(enricher *ebooks.Enricher) *SyncEbookMetadataTask {
|
||||
return &SyncEbookMetadataTask{enricher: enricher}
|
||||
}
|
||||
|
||||
func (t *SyncEbookMetadataTask) Key() string { return "sync_ebook_metadata" }
|
||||
func (t *SyncEbookMetadataTask) Name() string { return "Sync Ebook Metadata" }
|
||||
func (t *SyncEbookMetadataTask) Description() string {
|
||||
return "Fetches metadata (cover art, overview, authors) for ebooks that have not yet been enriched"
|
||||
}
|
||||
func (t *SyncEbookMetadataTask) Category() taskmanager.TaskCategory {
|
||||
return taskmanager.TaskCategoryMetadata
|
||||
}
|
||||
func (t *SyncEbookMetadataTask) IsHidden() bool { return false }
|
||||
func (t *SyncEbookMetadataTask) DefaultTriggers() []taskmanager.TriggerConfig {
|
||||
return []taskmanager.TriggerConfig{{Type: taskmanager.TriggerTypeInterval, IntervalMs: 5 * 60 * 1000}}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Wire `ebooks.NewEnricher` beside audiobook enricher**
|
||||
|
||||
In `cmd/silo/main.go`:
|
||||
- add import `github.com/Silo-Server/silo-server/internal/ebooks`
|
||||
- add `var ebookEnricher *ebooks.Enricher`
|
||||
- construct it with the same dependencies as `audiobooks.NewEnricher`
|
||||
- set image cacher if object storage is available and the ebook enricher supports remote image caching
|
||||
- register `tasks.NewSyncEbookMetadataTask(ebookEnricher)` beside `NewSyncAudiobookMetadataTask`
|
||||
|
||||
- [ ] **Step 3: Run task/main build tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/taskmanager/tasks ./internal/ebooks -count=1
|
||||
go test ./cmd/silo -run TestNonExistent -count=1
|
||||
```
|
||||
|
||||
Expected: packages compile and tests pass.
|
||||
|
||||
- [ ] **Step 4: Commit Task 5**
|
||||
|
||||
```bash
|
||||
git add cmd/silo/main.go internal/taskmanager/tasks/sync_ebook_metadata.go
|
||||
git commit -m "feat: wire ebook metadata sync task"
|
||||
```
|
||||
|
||||
### Task 6: Verification And PR Cleanup
|
||||
|
||||
**Files:**
|
||||
- Review all modified files.
|
||||
|
||||
- [ ] **Step 1: Run focused Go tests**
|
||||
|
||||
```bash
|
||||
go test ./internal/scanner ./internal/ebooks ./internal/taskmanager/tasks -count=1
|
||||
```
|
||||
|
||||
Expected: pass.
|
||||
|
||||
- [ ] **Step 2: Run broader compile/test set**
|
||||
|
||||
```bash
|
||||
go test ./internal/catalog ./internal/metadata ./cmd/silo -count=1
|
||||
```
|
||||
|
||||
Expected: pass, or record unrelated baseline failures with exact package/error.
|
||||
|
||||
- [ ] **Step 3: Check no wrong architecture remains**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
rg -n "ebook_details|silo-plugin-ebooks|PersonKindNarrator|asin" internal/scanner internal/ebooks docs/superpowers/specs docs/superpowers/plans
|
||||
```
|
||||
|
||||
Expected:
|
||||
- no `ebook_details`
|
||||
- no scanner dependency on `silo-plugin-ebooks`
|
||||
- no ebook code writing `PersonKindNarrator`
|
||||
- no ebook code writing ASIN
|
||||
|
||||
- [ ] **Step 4: Commit final cleanup if needed**
|
||||
|
||||
```bash
|
||||
git status --short
|
||||
git add <only ebook-related cleanup files>
|
||||
git commit -m "test: verify ebook audiobook-parity path"
|
||||
```
|
||||
|
||||
Only commit if Step 1-3 caused file changes.
|
||||
@@ -0,0 +1,102 @@
|
||||
# Ebooks Like Audiobooks Design
|
||||
|
||||
## Goal
|
||||
|
||||
Add ebooks to silo by matching the audiobook architecture in `main` as closely
|
||||
as possible. Ebooks are a core media type in `silo-server`; only metadata
|
||||
provider lookup lives in a plugin.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not add or keep a scanner/runtime plugin named `silo-plugin-ebooks`.
|
||||
- Do not treat ebooks as an audiobook sub-feature.
|
||||
- Do not add ebook narrators.
|
||||
- Do not add ebook ASIN handling.
|
||||
- Do not introduce an ebook details repository/table unless the audiobook path
|
||||
gains the same shape first.
|
||||
|
||||
## Audiobook Pattern In Main
|
||||
|
||||
Audiobooks have three cooperating pieces:
|
||||
|
||||
1. Core scan:
|
||||
- `internal/scanner/scanner.go` routes `audiobook` and `audiobooks` library
|
||||
types to `ScanAudiobookFolder`.
|
||||
- `internal/scanner/audiobook_scan.go` writes `media_items`,
|
||||
`media_files`, `item_people`, `audiobook_series`,
|
||||
`media_item_libraries`, and durable provider IDs.
|
||||
2. Core metadata enrichment:
|
||||
- `cmd/silo/main.go` wires `audiobooks.NewEnricher`.
|
||||
- `internal/taskmanager/tasks/sync_audiobook_metadata.go` registers the
|
||||
periodic/manual task.
|
||||
- `internal/audiobooks/enrichment.go` sweeps unenriched `audiobook` items,
|
||||
resolves the configured metadata provider chain at
|
||||
`content_level = 'audiobook'`, calls `Search` then `GetMetadata`, and
|
||||
persists provider IDs, poster/overview/scalar metadata, and people.
|
||||
3. First-party metadata plugin:
|
||||
- `silo-plugin-audiobook-metadata` exposes `metadata_provider.v1` with an
|
||||
audiobook capability and provider implementations.
|
||||
|
||||
## Ebook Equivalent Across PRs
|
||||
|
||||
Ebooks should mirror that shape:
|
||||
|
||||
1. Core scan:
|
||||
- `internal/scanner/scanner.go` routes `ebook` and `ebooks` libraries to an
|
||||
ebook scan path.
|
||||
- `internal/scanner/ebook.go` parses local ebook metadata from supported
|
||||
files.
|
||||
- `internal/scanner/ebook_scan.go` writes:
|
||||
- `media_items.type = 'ebook'`
|
||||
- `media_files.base_type = 'ebook'`
|
||||
- `item_people` author credits only
|
||||
- `ebook_series`
|
||||
- `media_item_libraries`
|
||||
- `media_item_provider_ids` using provider `isbn` when an ISBN is parsed
|
||||
2. Follow-up core metadata enrichment:
|
||||
- Add `internal/ebooks/enrichment.go`, shaped after
|
||||
`internal/audiobooks/enrichment.go`.
|
||||
- Add `internal/taskmanager/tasks/sync_ebook_metadata.go`, shaped after
|
||||
`sync_audiobook_metadata.go`.
|
||||
- Wire `ebooks.NewEnricher` in `cmd/silo/main.go` next to the audiobook
|
||||
enricher.
|
||||
- Resolve provider chains at `content_level = 'ebook'`.
|
||||
- Search and fetch metadata with `ContentType = "ebook"`.
|
||||
- Persist provider IDs, poster/overview/scalar metadata, and author people.
|
||||
3. First-party metadata plugin:
|
||||
- Use `silo-plugin-ebook-metadata` as the first-party ebook metadata
|
||||
provider plugin.
|
||||
- The core app should rely on that plugin for provider-specific metadata
|
||||
lookups, just as audiobooks rely on `silo-plugin-audiobook-metadata`.
|
||||
|
||||
## Ebook Domain Rules
|
||||
|
||||
- Ebooks have ISBNs.
|
||||
- Ebooks have authors.
|
||||
- Ebooks do not have narrators.
|
||||
- Ebooks do not have ASINs.
|
||||
- Ebook metadata plugins may return provider IDs, but core scanner-derived ISBN
|
||||
should be stored under provider `isbn`.
|
||||
- Local parsing may read embedded ebook metadata from EPUB/FB2 and format/file
|
||||
facts from other supported ebook formats.
|
||||
|
||||
## First PR Scope
|
||||
|
||||
The first implementation PR should establish the mirrored foundation:
|
||||
|
||||
- ebook library type and walk mode
|
||||
- local ebook parser
|
||||
- ebook scan routing and persistence
|
||||
- `ebook_series` migration
|
||||
- tests proving ebook authors-only behavior, ISBN provider ID behavior, and
|
||||
scanner-owned ebook series behavior
|
||||
|
||||
Deferred follow-up PR scope:
|
||||
|
||||
- ebook metadata enrichment package/task/wiring, analogous to
|
||||
`internal/audiobooks.Enricher`
|
||||
- plugin chain resolution at `content_level = 'ebook'`
|
||||
|
||||
UI/detail page, ebook reader experience, ABS ebook compatibility endpoints, and
|
||||
advanced provider-specific plugin improvements can be separate PRs unless they
|
||||
are required to prove the core path works.
|
||||
@@ -0,0 +1,377 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maxEPUBMetadataEntrySize = 8 * 1024 * 1024
|
||||
|
||||
var ebookExtensions = map[string]bool{
|
||||
".epub": true,
|
||||
".pdf": true,
|
||||
".mobi": true,
|
||||
".azw": true,
|
||||
".azw3": true,
|
||||
".fb2": true,
|
||||
}
|
||||
|
||||
type parsedEbook struct {
|
||||
Format string
|
||||
Title string
|
||||
Authors []string
|
||||
Description string
|
||||
Publisher string
|
||||
PublishedAt time.Time
|
||||
Year int
|
||||
Language string
|
||||
ISBN string
|
||||
Series string
|
||||
SeriesIndex string
|
||||
Genres []string
|
||||
PageCount int
|
||||
Cover *parsedEbookCover
|
||||
}
|
||||
|
||||
type parsedEbookCover struct {
|
||||
ContentType string
|
||||
Bytes []byte
|
||||
}
|
||||
|
||||
func SupportsEbookFile(filePath string) bool {
|
||||
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
|
||||
}
|
||||
|
||||
func parseEbookFile(path string) (book parsedEbook, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic parsing ebook %s: %v", path, r)
|
||||
}
|
||||
}()
|
||||
switch strings.ToLower(filepath.Ext(path)) {
|
||||
case ".epub":
|
||||
book, err = parseEbookEPUB(path)
|
||||
case ".fb2":
|
||||
book, err = parseEbookFB2(path)
|
||||
case ".pdf", ".mobi", ".azw", ".azw3":
|
||||
book = parsedEbook{Format: strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")}
|
||||
default:
|
||||
err = fmt.Errorf("unsupported ebook format: %s", filepath.Ext(path))
|
||||
}
|
||||
book.sanitize()
|
||||
return book, err
|
||||
}
|
||||
|
||||
func (b *parsedEbook) sanitize() {
|
||||
b.Format = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(b.Format)), ".")
|
||||
b.Title = strings.TrimSpace(b.Title)
|
||||
b.Description = strings.TrimSpace(b.Description)
|
||||
b.Publisher = strings.TrimSpace(b.Publisher)
|
||||
b.Language = strings.TrimSpace(b.Language)
|
||||
b.ISBN = normalizeEbookISBN(b.ISBN)
|
||||
b.Series = strings.TrimSpace(b.Series)
|
||||
b.SeriesIndex = strings.TrimSpace(b.SeriesIndex)
|
||||
b.Authors = uniqueTrimmedStrings(b.Authors)
|
||||
b.Genres = uniqueTrimmedStrings(b.Genres)
|
||||
if b.PageCount < 0 {
|
||||
b.PageCount = 0
|
||||
}
|
||||
if b.Year == 0 && !b.PublishedAt.IsZero() {
|
||||
b.Year = b.PublishedAt.Year()
|
||||
}
|
||||
}
|
||||
|
||||
func uniqueTrimmedStrings(values []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
var out []string
|
||||
for _, value := range values {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
key := strings.ToLower(trimmed)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, trimmed)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeEbookISBN(value string) string {
|
||||
value = strings.ToUpper(strings.TrimSpace(value))
|
||||
for _, prefix := range []string{"ISBN-13", "ISBN-10", "ISBN"} {
|
||||
if strings.HasPrefix(value, prefix) {
|
||||
value = strings.TrimSpace(strings.TrimPrefix(value, prefix))
|
||||
value = strings.TrimLeft(value, ": ")
|
||||
break
|
||||
}
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, r := range value {
|
||||
if r >= '0' && r <= '9' {
|
||||
out.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
if r == 'X' {
|
||||
out.WriteRune(r)
|
||||
}
|
||||
}
|
||||
candidate := out.String()
|
||||
switch len(candidate) {
|
||||
case 10:
|
||||
if validISBN10(candidate) {
|
||||
return candidate
|
||||
}
|
||||
case 13:
|
||||
if validISBN13(candidate) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func validISBN10(candidate string) bool {
|
||||
if len(candidate) != 10 {
|
||||
return false
|
||||
}
|
||||
sum := 0
|
||||
for i, r := range candidate {
|
||||
value := 0
|
||||
switch {
|
||||
case r >= '0' && r <= '9':
|
||||
value = int(r - '0')
|
||||
case r == 'X' && i == 9:
|
||||
value = 10
|
||||
default:
|
||||
return false
|
||||
}
|
||||
sum += value * (10 - i)
|
||||
}
|
||||
return sum%11 == 0
|
||||
}
|
||||
|
||||
func validISBN13(candidate string) bool {
|
||||
if len(candidate) != 13 {
|
||||
return false
|
||||
}
|
||||
sum := 0
|
||||
for i, r := range candidate {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
value := int(r - '0')
|
||||
if i%2 == 1 {
|
||||
value *= 3
|
||||
}
|
||||
sum += value
|
||||
}
|
||||
return sum%10 == 0
|
||||
}
|
||||
|
||||
func parseEbookEPUB(path string) (parsedEbook, error) {
|
||||
book := parsedEbook{Format: "epub"}
|
||||
reader, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return book, err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
container, err := readEPUBZipEntry(&reader.Reader, "META-INF/container.xml")
|
||||
if err != nil {
|
||||
return book, err
|
||||
}
|
||||
opfPath, err := epubOPFPath(container)
|
||||
if err != nil {
|
||||
return book, err
|
||||
}
|
||||
opf, err := readEPUBZipEntry(&reader.Reader, opfPath)
|
||||
if err != nil {
|
||||
return book, err
|
||||
}
|
||||
if err := parseEPUBOPFMetadata(opf, &book); err != nil {
|
||||
return book, err
|
||||
}
|
||||
return book, nil
|
||||
}
|
||||
|
||||
func parseEbookFB2(path string) (parsedEbook, error) {
|
||||
book := parsedEbook{Format: "fb2"}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return book, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var fb2 struct {
|
||||
Description struct {
|
||||
TitleInfo struct {
|
||||
Genres []string `xml:"genre"`
|
||||
Authors []struct {
|
||||
FirstName string `xml:"first-name"`
|
||||
MiddleName string `xml:"middle-name"`
|
||||
LastName string `xml:"last-name"`
|
||||
Nickname string `xml:"nickname"`
|
||||
} `xml:"author"`
|
||||
BookTitle string `xml:"book-title"`
|
||||
Lang string `xml:"lang"`
|
||||
Date struct {
|
||||
Value string `xml:"value,attr"`
|
||||
Text string `xml:",chardata"`
|
||||
} `xml:"date"`
|
||||
Sequences []struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Number string `xml:"number,attr"`
|
||||
} `xml:"sequence"`
|
||||
} `xml:"title-info"`
|
||||
PublishInfo struct {
|
||||
ISBN string `xml:"isbn"`
|
||||
Publisher string `xml:"publisher"`
|
||||
Year string `xml:"year"`
|
||||
} `xml:"publish-info"`
|
||||
} `xml:"description"`
|
||||
}
|
||||
if err := xml.NewDecoder(file).Decode(&fb2); err != nil {
|
||||
return book, err
|
||||
}
|
||||
|
||||
book.Title = fb2.Description.TitleInfo.BookTitle
|
||||
book.Language = fb2.Description.TitleInfo.Lang
|
||||
book.Genres = fb2.Description.TitleInfo.Genres
|
||||
if t, ok := parseEbookDate(firstNonEmpty(fb2.Description.TitleInfo.Date.Value, fb2.Description.TitleInfo.Date.Text)); ok {
|
||||
book.PublishedAt = t
|
||||
}
|
||||
for _, author := range fb2.Description.TitleInfo.Authors {
|
||||
name := strings.Join(uniqueTrimmedStrings([]string{
|
||||
author.FirstName,
|
||||
author.MiddleName,
|
||||
author.LastName,
|
||||
}), " ")
|
||||
if name == "" {
|
||||
name = author.Nickname
|
||||
}
|
||||
book.Authors = append(book.Authors, name)
|
||||
}
|
||||
if len(fb2.Description.TitleInfo.Sequences) > 0 {
|
||||
book.Series = fb2.Description.TitleInfo.Sequences[0].Name
|
||||
book.SeriesIndex = fb2.Description.TitleInfo.Sequences[0].Number
|
||||
}
|
||||
book.ISBN = fb2.Description.PublishInfo.ISBN
|
||||
book.Publisher = fb2.Description.PublishInfo.Publisher
|
||||
if year, err := strconv.Atoi(strings.TrimSpace(fb2.Description.PublishInfo.Year)); err == nil {
|
||||
book.Year = year
|
||||
}
|
||||
return book, nil
|
||||
}
|
||||
|
||||
func readEPUBZipEntry(reader *zip.Reader, name string) ([]byte, error) {
|
||||
for _, file := range reader.File {
|
||||
if file.Name != name {
|
||||
continue
|
||||
}
|
||||
entry, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer entry.Close()
|
||||
limited := io.LimitReader(entry, maxEPUBMetadataEntrySize+1)
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > maxEPUBMetadataEntrySize {
|
||||
return nil, fmt.Errorf("epub entry too large: %s", name)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
return nil, fmt.Errorf("epub entry not found: %s", name)
|
||||
}
|
||||
|
||||
func epubOPFPath(container []byte) (string, error) {
|
||||
var parsed struct {
|
||||
Rootfiles []struct {
|
||||
FullPath string `xml:"full-path,attr"`
|
||||
} `xml:"rootfiles>rootfile"`
|
||||
}
|
||||
if err := xml.Unmarshal(container, &parsed); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, rootfile := range parsed.Rootfiles {
|
||||
if strings.TrimSpace(rootfile.FullPath) != "" {
|
||||
return rootfile.FullPath, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("epub container has no rootfile")
|
||||
}
|
||||
|
||||
func parseEPUBOPFMetadata(opf []byte, book *parsedEbook) error {
|
||||
var parsed struct {
|
||||
Metadata struct {
|
||||
Titles []string `xml:"title"`
|
||||
Creators []string `xml:"creator"`
|
||||
Identifiers []string `xml:"identifier"`
|
||||
Publisher string `xml:"publisher"`
|
||||
Dates []string `xml:"date"`
|
||||
Language string `xml:"language"`
|
||||
Subjects []string `xml:"subject"`
|
||||
Descriptions []string `xml:"description"`
|
||||
Meta []struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Property string `xml:"property,attr"`
|
||||
Content string `xml:"content,attr"`
|
||||
Value string `xml:",chardata"`
|
||||
} `xml:"meta"`
|
||||
} `xml:"metadata"`
|
||||
}
|
||||
if err := xml.Unmarshal(opf, &parsed); err != nil {
|
||||
return err
|
||||
}
|
||||
book.Title = firstNonEmpty(parsed.Metadata.Titles...)
|
||||
book.Authors = append(book.Authors, parsed.Metadata.Creators...)
|
||||
book.Publisher = parsed.Metadata.Publisher
|
||||
book.Language = parsed.Metadata.Language
|
||||
book.Genres = append(book.Genres, parsed.Metadata.Subjects...)
|
||||
book.Description = firstNonEmpty(parsed.Metadata.Descriptions...)
|
||||
for _, identifier := range parsed.Metadata.Identifiers {
|
||||
if isbn := normalizeEbookISBN(identifier); isbn != "" {
|
||||
book.ISBN = isbn
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, date := range parsed.Metadata.Dates {
|
||||
if t, ok := parseEbookDate(date); ok {
|
||||
book.PublishedAt = t
|
||||
book.Year = t.Year()
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, meta := range parsed.Metadata.Meta {
|
||||
name := strings.ToLower(strings.TrimSpace(firstNonEmpty(meta.Name, meta.Property)))
|
||||
value := strings.TrimSpace(firstNonEmpty(meta.Content, meta.Value))
|
||||
switch name {
|
||||
case "calibre:series", "belongs-to-collection":
|
||||
book.Series = value
|
||||
case "calibre:series_index", "group-position":
|
||||
book.SeriesIndex = value
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseEbookDate(value string) (time.Time, bool) {
|
||||
value = strings.TrimSpace(value)
|
||||
for _, layout := range []string{"2006-01-02", "2006-01", "2006"} {
|
||||
if t, err := time.Parse(layout, value); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
@@ -0,0 +1,686 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/idgen"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/titleutil"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
type ebookSQLExecutor interface {
|
||||
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||
}
|
||||
|
||||
type filesystemMediaItemReader interface {
|
||||
GetByIDs(ctx context.Context, ids []string) ([]*models.MediaItem, error)
|
||||
}
|
||||
|
||||
func ebookScanWorkers() int {
|
||||
if v := os.Getenv("SILO_EBOOK_SCAN_WORKERS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return audiobookScanWorkers()
|
||||
}
|
||||
|
||||
func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolder) error {
|
||||
if s == nil || folder == nil {
|
||||
return fmt.Errorf("ScanEbookFolder: nil scanner or folder")
|
||||
}
|
||||
|
||||
var candidates []string
|
||||
for _, root := range folder.Paths {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if walkErr != nil {
|
||||
slog.Warn("ebook scan: walk error", "path", path, "error", walkErr)
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if SupportsEbookFile(path) {
|
||||
candidates = append(candidates, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if walkErr != nil {
|
||||
slog.Warn("ebook scan: walk root failed", "root", root, "error", walkErr)
|
||||
}
|
||||
}
|
||||
|
||||
if len(candidates) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
workers := ebookScanWorkers()
|
||||
slog.Info("ebook scan: starting",
|
||||
"folder_id", folder.ID,
|
||||
"candidates", len(candidates),
|
||||
"workers", workers,
|
||||
)
|
||||
|
||||
ch := make(chan string, workers*2)
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
processed int64
|
||||
failed int64
|
||||
skipped int64
|
||||
failMu sync.Mutex
|
||||
failures []error
|
||||
cancelErr error
|
||||
)
|
||||
start := time.Now()
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for path := range ch {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := s.reconcileEbookFile(ctx, folder, path, &skipped); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
failMu.Lock()
|
||||
if cancelErr == nil {
|
||||
cancelErr = err
|
||||
}
|
||||
failMu.Unlock()
|
||||
return
|
||||
}
|
||||
atomic.AddInt64(&failed, 1)
|
||||
failMu.Lock()
|
||||
failures = append(failures, fmt.Errorf("%s: %w", path, err))
|
||||
failMu.Unlock()
|
||||
slog.Warn("ebook scan: file failed",
|
||||
"folder_id", folder.ID,
|
||||
"path", path,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
n := atomic.AddInt64(&processed, 1)
|
||||
if n%500 == 0 {
|
||||
slog.Info("ebook scan: progress",
|
||||
"folder_id", folder.ID,
|
||||
"processed", n,
|
||||
"failed", atomic.LoadInt64(&failed),
|
||||
"skipped", atomic.LoadInt64(&skipped),
|
||||
"total", len(candidates),
|
||||
"elapsed_sec", int(time.Since(start).Seconds()),
|
||||
)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for _, p := range candidates {
|
||||
select {
|
||||
case ch <- p:
|
||||
case <-ctx.Done():
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if cancelErr != nil {
|
||||
return cancelErr
|
||||
}
|
||||
|
||||
slog.Info("ebook scan: completed",
|
||||
"folder_id", folder.ID,
|
||||
"processed", atomic.LoadInt64(&processed),
|
||||
"failed", atomic.LoadInt64(&failed),
|
||||
"skipped", atomic.LoadInt64(&skipped),
|
||||
"elapsed_sec", int(time.Since(start).Seconds()),
|
||||
)
|
||||
if processedCount := atomic.LoadInt64(&processed); processedCount > 0 {
|
||||
failedCount := atomic.LoadInt64(&failed)
|
||||
skippedCount := atomic.LoadInt64(&skipped)
|
||||
if failedCount > 0 && skippedCount == 0 && failedCount == processedCount {
|
||||
return fmt.Errorf("ebook scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) reconcileEbookFile(ctx context.Context, folder *models.MediaFolder, filePath string, skipped *int64) error {
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("stat ebook file %s: %w", filePath, err)
|
||||
}
|
||||
size := info.Size()
|
||||
modifiedAt := normalizeFileModifiedAt(info.ModTime())
|
||||
|
||||
isUnchanged, skipErr := s.ebookFileShouldSkip(ctx, folder, filePath, size, modifiedAt)
|
||||
if skipErr != nil {
|
||||
slog.Warn("ebook scan: skip-check failed, falling through",
|
||||
"folder_id", folder.ID,
|
||||
"path", filePath,
|
||||
"error", skipErr,
|
||||
)
|
||||
} else if isUnchanged {
|
||||
atomic.AddInt64(skipped, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
parsed, err := parseEbookFile(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse ebook file %s: %w", filePath, err)
|
||||
}
|
||||
if parsed.Title == "" {
|
||||
parsed.Title = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
|
||||
}
|
||||
|
||||
contentID, err := s.upsertEbookMediaItem(ctx, folder.ID, filePath, &parsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert ebook item: %w", err)
|
||||
}
|
||||
if err := s.upsertEbookMediaFile(ctx, folder, contentID, filePath, size, modifiedAt, &parsed); err != nil {
|
||||
return fmt.Errorf("upsert ebook file: %w", err)
|
||||
}
|
||||
if err := s.upsertEbookPeople(ctx, contentID, &parsed); err != nil {
|
||||
return fmt.Errorf("upsert ebook people: %w", err)
|
||||
}
|
||||
if err := s.upsertEbookSeries(ctx, contentID, &parsed); err != nil {
|
||||
return fmt.Errorf("upsert ebook series: %w", err)
|
||||
}
|
||||
if err := insertEbookLibraryMembership(ctx, s.fileRepo.Pool(), contentID, folder.ID); err != nil {
|
||||
return fmt.Errorf("upsert ebook library membership: %w", err)
|
||||
}
|
||||
if parsed.ISBN != "" {
|
||||
if err := insertEbookISBNProviderID(ctx, s.fileRepo.Pool(), contentID, parsed.ISBN); err != nil {
|
||||
return fmt.Errorf("upsert ebook ISBN provider id: %w", err)
|
||||
}
|
||||
}
|
||||
slog.Info("ebook scan: indexed",
|
||||
"folder_id", folder.ID,
|
||||
"content_id", contentID,
|
||||
"title", parsed.Title,
|
||||
"authors", parsed.Authors,
|
||||
"path", filePath,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) ebookFileShouldSkip(ctx context.Context, folder *models.MediaFolder, filePath string, size int64, modifiedAt time.Time) (bool, error) {
|
||||
if s.fileRepo == nil || s.itemRepo == nil {
|
||||
return false, nil
|
||||
}
|
||||
existing, err := s.fileRepo.ListByObservedRootPath(ctx, folder.ID, filePath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("list existing files: %w", err)
|
||||
}
|
||||
if len(existing) != 1 {
|
||||
return false, nil
|
||||
}
|
||||
mf := existing[0]
|
||||
if mf.FilePath != filePath || mf.FileSize != size || mf.FileModifiedAt == nil || !sameFileModifiedAt(mf.FileModifiedAt, modifiedAt) {
|
||||
return false, nil
|
||||
}
|
||||
if mf.ContentID == "" {
|
||||
return false, nil
|
||||
}
|
||||
statuses, err := s.itemRepo.GetStatusByIDs(ctx, []string{mf.ContentID})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("get item status: %w", err)
|
||||
}
|
||||
return !strings.EqualFold(strings.TrimSpace(statuses[mf.ContentID]), "unmatched"), nil
|
||||
}
|
||||
|
||||
func (s *Scanner) upsertEbookMediaItem(ctx context.Context, folderID int, filePath string, book *parsedEbook) (string, error) {
|
||||
if s.itemRepo == nil {
|
||||
return "", fmt.Errorf("itemRepo not configured on Scanner")
|
||||
}
|
||||
if s.fileRepo == nil {
|
||||
return "", fmt.Errorf("fileRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
existingID, err := s.fileRepo.FindContentIDByRootPath(ctx, folderID, filePath, "ebook")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find ebook by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
if err := updateExistingEbookMediaItem(ctx, s.itemRepo, s.itemRepo, existingID, book); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return existingID, nil
|
||||
}
|
||||
if existing := s.findEbookByFilePath(ctx, filePath); existing != nil {
|
||||
applyEbookToMediaItem(existing, book)
|
||||
if existing.SortTitle == "" {
|
||||
existing.SortTitle = titleutil.DeriveDefaultSortTitle(existing.Title)
|
||||
}
|
||||
if err := s.itemRepo.Upsert(ctx, existing); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return existing.ContentID, nil
|
||||
}
|
||||
return resolveEbookMediaItem(ctx, s.fileRepo, s.itemRepo, folderID, filePath, book)
|
||||
}
|
||||
|
||||
func resolveEbookMediaItem(
|
||||
ctx context.Context,
|
||||
rootFinder filesystemRootContentFinder,
|
||||
itemWriter filesystemMediaItemWriter,
|
||||
folderID int,
|
||||
filePath string,
|
||||
book *parsedEbook,
|
||||
) (string, error) {
|
||||
if rootFinder == nil {
|
||||
return "", fmt.Errorf("root content finder not configured")
|
||||
}
|
||||
if itemWriter == nil {
|
||||
return "", fmt.Errorf("media item writer not configured")
|
||||
}
|
||||
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, filePath, "ebook")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find ebook by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
cleanTitle := strings.TrimSpace(book.Title)
|
||||
if cleanTitle == "" {
|
||||
cleanTitle = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
|
||||
}
|
||||
return createEbookMediaItem(ctx, itemWriter, book, cleanTitle)
|
||||
}
|
||||
|
||||
func createEbookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedEbook, cleanTitle string) (string, error) {
|
||||
id, err := idgen.NextID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("generate content_id: %w", err)
|
||||
}
|
||||
item := &models.MediaItem{
|
||||
ContentID: id,
|
||||
Type: "ebook",
|
||||
Title: cleanTitle,
|
||||
SortTitle: titleutil.DeriveDefaultSortTitle(cleanTitle),
|
||||
Year: book.Year,
|
||||
}
|
||||
applyEbookToMediaItem(item, book)
|
||||
if item.SortTitle == "" {
|
||||
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
|
||||
}
|
||||
if err := itemWriter.Upsert(ctx, item); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func updateExistingEbookMediaItem(ctx context.Context, itemReader filesystemMediaItemReader, itemWriter filesystemMediaItemWriter, contentID string, book *parsedEbook) error {
|
||||
if itemReader == nil {
|
||||
return fmt.Errorf("media item reader not configured")
|
||||
}
|
||||
if itemWriter == nil {
|
||||
return fmt.Errorf("media item writer not configured")
|
||||
}
|
||||
items, err := itemReader.GetByIDs(ctx, []string{contentID})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get ebook media item %s: %w", contentID, err)
|
||||
}
|
||||
if len(items) == 0 || items[0] == nil {
|
||||
return fmt.Errorf("ebook media item %s not found", contentID)
|
||||
}
|
||||
item := items[0]
|
||||
applyEbookToMediaItem(item, book)
|
||||
if item.SortTitle == "" {
|
||||
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
|
||||
}
|
||||
return itemWriter.Upsert(ctx, item)
|
||||
}
|
||||
|
||||
func applyEbookToMediaItem(item *models.MediaItem, book *parsedEbook) {
|
||||
item.Type = "ebook"
|
||||
if title := strings.TrimSpace(book.Title); title != "" {
|
||||
item.Title = title
|
||||
}
|
||||
if book.Year > 0 {
|
||||
item.Year = book.Year
|
||||
}
|
||||
if book.Description != "" && item.Overview == "" {
|
||||
item.Overview = book.Description
|
||||
}
|
||||
if book.Publisher != "" {
|
||||
item.Studios = mergeUniqueStrings(item.Studios, []string{book.Publisher})
|
||||
}
|
||||
if len(book.Genres) > 0 {
|
||||
item.Genres = mergeUniqueStrings(item.Genres, book.Genres)
|
||||
}
|
||||
if !book.PublishedAt.IsZero() && item.ReleaseDate == nil {
|
||||
rd := book.PublishedAt.UTC().Format("2006-01-02")
|
||||
item.ReleaseDate = &rd
|
||||
}
|
||||
if book.Language != "" && item.OriginalLanguage == "" {
|
||||
item.OriginalLanguage = book.Language
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) findEbookByFilePath(ctx context.Context, filePath string) *models.MediaItem {
|
||||
if s.fileRepo == nil {
|
||||
return nil
|
||||
}
|
||||
var existingID string
|
||||
err := s.fileRepo.Pool().QueryRow(ctx, `
|
||||
SELECT mf.content_id
|
||||
FROM media_files mf
|
||||
JOIN media_items mi ON mi.content_id = mf.content_id
|
||||
WHERE mf.file_path = $1
|
||||
AND mi.type = 'ebook'
|
||||
LIMIT 1
|
||||
`, filePath).Scan(&existingID)
|
||||
if err != nil || existingID == "" {
|
||||
return nil
|
||||
}
|
||||
items, err := s.itemRepo.GetByIDs(ctx, []string{existingID})
|
||||
if err != nil || len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
return items[0]
|
||||
}
|
||||
|
||||
func (s *Scanner) upsertEbookMediaFile(ctx context.Context, folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) error {
|
||||
mf := buildEbookMediaFile(folder, contentID, filePath, size, modifiedAt, book)
|
||||
if _, err := s.fileRepo.Upsert(ctx, mf); err != nil {
|
||||
return fmt.Errorf("upsert media file %s: %w", filePath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildEbookMediaFile(folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) models.MediaFile {
|
||||
return models.MediaFile{
|
||||
ContentID: contentID,
|
||||
MediaFolderID: folder.ID,
|
||||
CanonicalRootPath: filePath,
|
||||
ObservedRootPath: filePath,
|
||||
ContentGroupKey: contentID,
|
||||
GroupKeyVersion: 1,
|
||||
BaseTitle: book.Title,
|
||||
BaseYear: book.Year,
|
||||
BaseType: "ebook",
|
||||
IdentityConfidence: ebookIdentityConfidence(book),
|
||||
FilePath: filePath,
|
||||
FileSize: size,
|
||||
FileModifiedAt: &modifiedAt,
|
||||
Container: book.Format,
|
||||
ProbeSource: "local",
|
||||
}
|
||||
}
|
||||
|
||||
func insertEbookLibraryMembership(ctx context.Context, exec ebookSQLExecutor, contentID string, folderID int) error {
|
||||
_, err := exec.Exec(ctx, `
|
||||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||||
VALUES ($1, $2, NOW())
|
||||
ON CONFLICT (content_id, media_folder_id) DO NOTHING
|
||||
`, contentID, folderID)
|
||||
return err
|
||||
}
|
||||
|
||||
func insertEbookISBNProviderID(ctx context.Context, exec ebookSQLExecutor, contentID string, isbn string) error {
|
||||
_, err := exec.Exec(ctx, `
|
||||
INSERT INTO media_item_provider_ids (content_id, provider, provider_id, item_type)
|
||||
VALUES ($1, 'isbn', $2, 'ebook')
|
||||
ON CONFLICT DO NOTHING
|
||||
`, contentID, isbn)
|
||||
return err
|
||||
}
|
||||
|
||||
type ebookCredit struct {
|
||||
Name string
|
||||
Kind models.PersonKind
|
||||
}
|
||||
|
||||
func ebookPeopleCreditsEqual(existing []models.ItemPerson, desired []ebookCredit) bool {
|
||||
existingAuthors := make([]models.ItemPerson, 0, len(existing))
|
||||
for _, p := range existing {
|
||||
if p.Kind == models.PersonKindAuthor {
|
||||
existingAuthors = append(existingAuthors, p)
|
||||
}
|
||||
}
|
||||
if len(existingAuthors) != len(desired) {
|
||||
return false
|
||||
}
|
||||
type key struct {
|
||||
name string
|
||||
kind models.PersonKind
|
||||
}
|
||||
have := make(map[key]struct{}, len(existingAuthors))
|
||||
for _, p := range existingAuthors {
|
||||
have[key{strings.ToLower(strings.TrimSpace(p.Person.Name)), p.Kind}] = struct{}{}
|
||||
}
|
||||
for _, d := range desired {
|
||||
k := key{strings.ToLower(strings.TrimSpace(d.Name)), d.Kind}
|
||||
if _, ok := have[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type ebookResolvedAuthor struct {
|
||||
ID int64
|
||||
Name string
|
||||
}
|
||||
|
||||
func mergeEbookPeople(existing []models.ItemPerson, authors []ebookResolvedAuthor) []models.ItemPerson {
|
||||
people := make([]models.ItemPerson, 0, len(existing)+len(authors))
|
||||
for _, p := range existing {
|
||||
if p.Kind == models.PersonKindAuthor {
|
||||
continue
|
||||
}
|
||||
p.SortOrder = len(people)
|
||||
people = append(people, p)
|
||||
}
|
||||
for _, author := range authors {
|
||||
people = append(people, models.ItemPerson{
|
||||
Person: models.Person{ID: author.ID, Name: author.Name},
|
||||
Kind: models.PersonKindAuthor,
|
||||
SortOrder: len(people),
|
||||
})
|
||||
}
|
||||
return people
|
||||
}
|
||||
|
||||
func ebookPeopleForReplace(existing []models.ItemPerson, getErr error, authors []ebookResolvedAuthor) ([]models.ItemPerson, error) {
|
||||
if getErr != nil {
|
||||
return nil, fmt.Errorf("get ebook people: %w", getErr)
|
||||
}
|
||||
return mergeEbookPeople(existing, authors), nil
|
||||
}
|
||||
|
||||
func (s *Scanner) upsertEbookPeople(ctx context.Context, contentID string, book *parsedEbook) error {
|
||||
if s.personRepo == nil {
|
||||
return fmt.Errorf("personRepo not configured on Scanner")
|
||||
}
|
||||
if s.itemRepo == nil {
|
||||
return fmt.Errorf("itemRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
var desired []ebookCredit
|
||||
for _, author := range book.Authors {
|
||||
if name := strings.TrimSpace(author); name != "" {
|
||||
desired = append(desired, ebookCredit{Name: name, Kind: models.PersonKindAuthor})
|
||||
}
|
||||
}
|
||||
if len(desired) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
existing, err := s.itemRepo.GetPeople(ctx, contentID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get ebook people: %w", err)
|
||||
}
|
||||
if ebookPeopleCreditsEqual(existing, desired) {
|
||||
return nil
|
||||
}
|
||||
|
||||
authors := make([]ebookResolvedAuthor, 0, len(desired))
|
||||
for _, c := range desired {
|
||||
personID, err := s.personRepo.FindOrCreate(ctx, models.Person{Name: c.Name})
|
||||
if err != nil {
|
||||
return fmt.Errorf("find-or-create person %q: %w", c.Name, err)
|
||||
}
|
||||
authors = append(authors, ebookResolvedAuthor{ID: personID, Name: c.Name})
|
||||
}
|
||||
people, err := ebookPeopleForReplace(existing, nil, authors)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.itemRepo.ReplacePeople(ctx, contentID, people)
|
||||
}
|
||||
|
||||
func (s *Scanner) upsertEbookSeries(ctx context.Context, contentID string, book *parsedEbook) error {
|
||||
if s == nil {
|
||||
return fmt.Errorf("Scanner not configured")
|
||||
}
|
||||
if s.fileRepo == nil {
|
||||
return fmt.Errorf("fileRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
var currentName *string
|
||||
var currentIdx *float64
|
||||
err := s.fileRepo.Pool().QueryRow(ctx, `
|
||||
SELECT series_name, series_index FROM ebook_series WHERE content_id = $1
|
||||
`, contentID).Scan(¤tName, ¤tIdx)
|
||||
|
||||
plan, err := planEbookSeriesWrite(book, currentName, currentIdx, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch plan.Kind {
|
||||
case ebookSeriesWriteNone:
|
||||
return nil
|
||||
case ebookSeriesWriteDelete:
|
||||
if _, delErr := s.fileRepo.Pool().Exec(ctx,
|
||||
`DELETE FROM ebook_series WHERE content_id = $1`, contentID); delErr != nil {
|
||||
return fmt.Errorf("delete ebook_series row: %w", delErr)
|
||||
}
|
||||
return nil
|
||||
case ebookSeriesWriteUpsert:
|
||||
var idx any
|
||||
if plan.Index != nil {
|
||||
idx = *plan.Index
|
||||
}
|
||||
if _, err := s.fileRepo.Pool().Exec(ctx, `
|
||||
INSERT INTO ebook_series (content_id, series_name, series_index, updated_at)
|
||||
VALUES ($1, $2, $3, NOW())
|
||||
ON CONFLICT (content_id) DO UPDATE SET
|
||||
series_name = EXCLUDED.series_name,
|
||||
series_index = EXCLUDED.series_index,
|
||||
updated_at = NOW()
|
||||
`, contentID, plan.Name, idx); err != nil {
|
||||
return fmt.Errorf("upsert ebook_series row: %w", err)
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unknown ebook_series write kind: %d", plan.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func ebookSeriesDesired(book *parsedEbook) (string, *float64) {
|
||||
if book == nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(book.Series), parseSeriesIndex(book.SeriesIndex)
|
||||
}
|
||||
|
||||
type ebookSeriesWriteKind int
|
||||
|
||||
const (
|
||||
ebookSeriesWriteNone ebookSeriesWriteKind = iota
|
||||
ebookSeriesWriteDelete
|
||||
ebookSeriesWriteUpsert
|
||||
)
|
||||
|
||||
type ebookSeriesWritePlan struct {
|
||||
Kind ebookSeriesWriteKind
|
||||
Name string
|
||||
Index *float64
|
||||
}
|
||||
|
||||
func planEbookSeriesWrite(book *parsedEbook, currentName *string, currentIdx *float64, queryErr error) (ebookSeriesWritePlan, error) {
|
||||
if queryErr != nil {
|
||||
if !errors.Is(queryErr, pgx.ErrNoRows) {
|
||||
return ebookSeriesWritePlan{}, fmt.Errorf("query ebook_series: %w", queryErr)
|
||||
}
|
||||
currentName = nil
|
||||
currentIdx = nil
|
||||
}
|
||||
|
||||
desiredName, desiredIdx := ebookSeriesDesired(book)
|
||||
if desiredName == "" {
|
||||
if currentName == nil {
|
||||
return ebookSeriesWritePlan{Kind: ebookSeriesWriteNone}, nil
|
||||
}
|
||||
return ebookSeriesWritePlan{Kind: ebookSeriesWriteDelete}, nil
|
||||
}
|
||||
|
||||
if currentName != nil && *currentName == desiredName && floatPtrEqual(currentIdx, desiredIdx) {
|
||||
return ebookSeriesWritePlan{Kind: ebookSeriesWriteNone}, nil
|
||||
}
|
||||
|
||||
return ebookSeriesWritePlan{
|
||||
Kind: ebookSeriesWriteUpsert,
|
||||
Name: desiredName,
|
||||
Index: desiredIdx,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ebookIdentityConfidence(book *parsedEbook) string {
|
||||
if book == nil {
|
||||
return "low"
|
||||
}
|
||||
|
||||
score := 0
|
||||
if book.Title != "" {
|
||||
score++
|
||||
}
|
||||
if len(book.Authors) > 0 {
|
||||
score++
|
||||
}
|
||||
if book.Year > 0 {
|
||||
score++
|
||||
}
|
||||
if book.ISBN != "" {
|
||||
score++
|
||||
}
|
||||
|
||||
switch {
|
||||
case score >= 4:
|
||||
return "high"
|
||||
case score > 0:
|
||||
return "medium"
|
||||
default:
|
||||
return "low"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,590 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
type recordingEbookExecutor struct {
|
||||
queries []string
|
||||
args [][]any
|
||||
}
|
||||
|
||||
func (r *recordingEbookExecutor) Exec(_ context.Context, query string, args ...any) (pgconn.CommandTag, error) {
|
||||
r.queries = append(r.queries, query)
|
||||
r.args = append(r.args, append([]any(nil), args...))
|
||||
return pgconn.CommandTag{}, nil
|
||||
}
|
||||
|
||||
type fakeFilesystemItemReader struct {
|
||||
items []*models.MediaItem
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeFilesystemItemReader) GetByIDs(_ context.Context, _ []string) ([]*models.MediaItem, error) {
|
||||
return f.items, f.err
|
||||
}
|
||||
|
||||
func TestSupportsEbookFile(t *testing.T) {
|
||||
cases := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"book.epub", true},
|
||||
{"book.PDF", true},
|
||||
{"book.mobi", true},
|
||||
{"book.azw", true},
|
||||
{"book.azw3", true},
|
||||
{"book.FB2", true},
|
||||
{"book.mp3", false},
|
||||
{"movie.mkv", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := SupportsEbookFile(tc.path); got != tc.want {
|
||||
t.Errorf("SupportsEbookFile(%q) = %v, want %v", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeEbookISBNHandlesCommonLabels(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"ISBN-13: 978-0-306-40615-7", "9780306406157"},
|
||||
{"ISBN-10: 0-9752298-0-x", "097522980X"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := normalizeEbookISBN(tc.in); got != tc.want {
|
||||
t.Errorf("normalizeEbookISBN(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEbookEPUBMetadata(t *testing.T) {
|
||||
path := writeTestEPUB(t, []string{"ISBN: 978-0-306-40615-7"})
|
||||
|
||||
got, err := parseEbookFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("parseEbookFile: %v", err)
|
||||
}
|
||||
|
||||
if got.Format != "epub" {
|
||||
t.Fatalf("Format = %q, want epub", got.Format)
|
||||
}
|
||||
if got.Title != "The Test Ebook" {
|
||||
t.Fatalf("Title = %q, want The Test Ebook", got.Title)
|
||||
}
|
||||
if strings.Join(got.Authors, ", ") != "Ada Writer, Ben Author" {
|
||||
t.Fatalf("Authors = %v, want Ada Writer and Ben Author", got.Authors)
|
||||
}
|
||||
if got.ISBN != "9780306406157" {
|
||||
t.Fatalf("ISBN = %q, want normalized ISBN", got.ISBN)
|
||||
}
|
||||
if got.Year != 2024 {
|
||||
t.Fatalf("Year = %d, want 2024", got.Year)
|
||||
}
|
||||
if got.Publisher != "Silo Press" {
|
||||
t.Fatalf("Publisher = %q, want Silo Press", got.Publisher)
|
||||
}
|
||||
wantPublishedAt := time.Date(2024, 3, 10, 0, 0, 0, 0, time.UTC)
|
||||
if !got.PublishedAt.Equal(wantPublishedAt) {
|
||||
t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, wantPublishedAt)
|
||||
}
|
||||
if got.Language != "en" {
|
||||
t.Fatalf("Language = %q, want en", got.Language)
|
||||
}
|
||||
if strings.Join(got.Genres, ", ") != "Fiction, Adventure" {
|
||||
t.Fatalf("Genres = %v, want Fiction and Adventure", got.Genres)
|
||||
}
|
||||
if got.Series != "Test Series" || got.SeriesIndex != "2" {
|
||||
t.Fatalf("Series = %q/%q, want Test Series/2", got.Series, got.SeriesIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEbookEPUBSkipsUUIDIdentifierBeforeISBN(t *testing.T) {
|
||||
path := writeTestEPUB(t, []string{
|
||||
"550e8400-e29b-41d4-a716-446655440000",
|
||||
"ISBN: 978-0-306-40615-7",
|
||||
})
|
||||
|
||||
got, err := parseEbookFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("parseEbookFile: %v", err)
|
||||
}
|
||||
if got.ISBN != "9780306406157" {
|
||||
t.Fatalf("ISBN = %q, want normalized ISBN after skipping UUID", got.ISBN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadEPUBZipEntryRejectsOversizedEntry(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "large.epub")
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create epub: %v", err)
|
||||
}
|
||||
zw := zip.NewWriter(file)
|
||||
w, err := zw.Create("OPS/content.opf")
|
||||
if err != nil {
|
||||
t.Fatalf("create zip entry: %v", err)
|
||||
}
|
||||
if _, err := w.Write([]byte(strings.Repeat("x", maxEPUBMetadataEntrySize+1))); err != nil {
|
||||
t.Fatalf("write zip entry: %v", err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("close zip: %v", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatalf("close file: %v", err)
|
||||
}
|
||||
|
||||
reader, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open zip: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
_, err = readEPUBZipEntry(&reader.Reader, "OPS/content.opf")
|
||||
if err == nil || !strings.Contains(err.Error(), "epub entry too large") {
|
||||
t.Fatalf("readEPUBZipEntry error = %v, want size limit error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEbookFB2Metadata(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "book.fb2")
|
||||
if err := os.WriteFile(path, []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<FictionBook>
|
||||
<description>
|
||||
<title-info>
|
||||
<genre>science fiction</genre>
|
||||
<author><first-name>Ada</first-name><last-name>Writer</last-name></author>
|
||||
<book-title>FB2 Test Ebook</book-title>
|
||||
<date value="2022-04-05">5 April 2022</date>
|
||||
<lang>en</lang>
|
||||
<sequence name="FB2 Series" number="3"/>
|
||||
</title-info>
|
||||
<publish-info>
|
||||
<publisher>FB2 Press</publisher>
|
||||
<year>2022</year>
|
||||
<isbn>978-0-306-40615-7</isbn>
|
||||
</publish-info>
|
||||
</description>
|
||||
</FictionBook>`), 0o644); err != nil {
|
||||
t.Fatalf("write fb2: %v", err)
|
||||
}
|
||||
|
||||
got, err := parseEbookFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("parseEbookFile: %v", err)
|
||||
}
|
||||
|
||||
if got.Format != "fb2" || got.Title != "FB2 Test Ebook" {
|
||||
t.Fatalf("Format/Title = %q/%q, want fb2/FB2 Test Ebook", got.Format, got.Title)
|
||||
}
|
||||
if strings.Join(got.Authors, ", ") != "Ada Writer" {
|
||||
t.Fatalf("Authors = %v, want Ada Writer", got.Authors)
|
||||
}
|
||||
if got.ISBN != "9780306406157" {
|
||||
t.Fatalf("ISBN = %q, want normalized ISBN", got.ISBN)
|
||||
}
|
||||
if got.Publisher != "FB2 Press" {
|
||||
t.Fatalf("Publisher = %q, want FB2 Press", got.Publisher)
|
||||
}
|
||||
if got.Year != 2022 {
|
||||
t.Fatalf("Year = %d, want 2022", got.Year)
|
||||
}
|
||||
wantPublishedAt := time.Date(2022, 4, 5, 0, 0, 0, 0, time.UTC)
|
||||
if !got.PublishedAt.Equal(wantPublishedAt) {
|
||||
t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, wantPublishedAt)
|
||||
}
|
||||
if got.Language != "en" {
|
||||
t.Fatalf("Language = %q, want en", got.Language)
|
||||
}
|
||||
if strings.Join(got.Genres, ", ") != "science fiction" {
|
||||
t.Fatalf("Genres = %v, want science fiction", got.Genres)
|
||||
}
|
||||
if got.Series != "FB2 Series" || got.SeriesIndex != "3" {
|
||||
t.Fatalf("Series = %q/%q, want FB2 Series/3", got.Series, got.SeriesIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookSeriesDesiredParsesIndex(t *testing.T) {
|
||||
name, idx := ebookSeriesDesired(&parsedEbook{
|
||||
Series: " The Expanse ",
|
||||
SeriesIndex: "2 of 9",
|
||||
})
|
||||
|
||||
if name != "The Expanse" {
|
||||
t.Fatalf("series name = %q, want The Expanse", name)
|
||||
}
|
||||
if idx == nil || *idx != 2 {
|
||||
t.Fatalf("series index = %v, want 2", idx)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertEbookSeriesNilScannerReturnsError(t *testing.T) {
|
||||
var s *Scanner
|
||||
if err := s.upsertEbookSeries(context.Background(), "content-1", &parsedEbook{Series: "Series"}); err == nil {
|
||||
t.Fatal("upsertEbookSeries nil scanner error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertEbookSeriesNilFileRepoReturnsError(t *testing.T) {
|
||||
s := &Scanner{}
|
||||
if err := s.upsertEbookSeries(context.Background(), "content-1", &parsedEbook{Series: "Series"}); err == nil {
|
||||
t.Fatal("upsertEbookSeries nil fileRepo error = nil, want error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEbookSeriesWriteInsertsWhenRowAbsent(t *testing.T) {
|
||||
plan, err := planEbookSeriesWrite(&parsedEbook{Series: " Series ", SeriesIndex: "2 of 9"}, nil, nil, pgx.ErrNoRows)
|
||||
if err != nil {
|
||||
t.Fatalf("planEbookSeriesWrite: %v", err)
|
||||
}
|
||||
if plan.Kind != ebookSeriesWriteUpsert || plan.Name != "Series" {
|
||||
t.Fatalf("plan = %+v, want upsert Series", plan)
|
||||
}
|
||||
if plan.Index == nil || *plan.Index != 2 {
|
||||
t.Fatalf("index = %v, want 2", plan.Index)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEbookSeriesWriteBlankSeriesDeletesExistingAndSkipsAbsent(t *testing.T) {
|
||||
currentName := "Series"
|
||||
plan, err := planEbookSeriesWrite(&parsedEbook{Series: " "}, ¤tName, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("planEbookSeriesWrite delete: %v", err)
|
||||
}
|
||||
if plan.Kind != ebookSeriesWriteDelete {
|
||||
t.Fatalf("blank existing plan = %+v, want delete", plan)
|
||||
}
|
||||
|
||||
plan, err = planEbookSeriesWrite(&parsedEbook{Series: ""}, nil, nil, pgx.ErrNoRows)
|
||||
if err != nil {
|
||||
t.Fatalf("planEbookSeriesWrite absent: %v", err)
|
||||
}
|
||||
if plan.Kind != ebookSeriesWriteNone {
|
||||
t.Fatalf("blank absent plan = %+v, want none", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEbookSeriesWriteSkipsIdenticalRow(t *testing.T) {
|
||||
currentName := "Series"
|
||||
currentIdx := 3.5
|
||||
plan, err := planEbookSeriesWrite(&parsedEbook{Series: "Series", SeriesIndex: "3.5"}, ¤tName, ¤tIdx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("planEbookSeriesWrite: %v", err)
|
||||
}
|
||||
if plan.Kind != ebookSeriesWriteNone {
|
||||
t.Fatalf("identical plan = %+v, want none", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEbookSeriesWriteAllowsNullNumericIndex(t *testing.T) {
|
||||
plan, err := planEbookSeriesWrite(&parsedEbook{Series: "Series", SeriesIndex: "appendix"}, nil, nil, pgx.ErrNoRows)
|
||||
if err != nil {
|
||||
t.Fatalf("planEbookSeriesWrite: %v", err)
|
||||
}
|
||||
if plan.Kind != ebookSeriesWriteUpsert || plan.Index != nil {
|
||||
t.Fatalf("plan = %+v, want upsert with nil index", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanEbookSeriesWriteReturnsQueryError(t *testing.T) {
|
||||
queryErr := errors.New("query failed")
|
||||
if _, err := planEbookSeriesWrite(&parsedEbook{Series: "Series"}, nil, nil, queryErr); !errors.Is(err, queryErr) {
|
||||
t.Fatalf("planEbookSeriesWrite error = %v, want query error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
|
||||
book := &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
|
||||
if got := ebookIdentityConfidence(book); got != "high" {
|
||||
t.Fatalf("complete metadata confidence = %q, want high", got)
|
||||
}
|
||||
|
||||
book = &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024}
|
||||
if got := ebookIdentityConfidence(book); got != "medium" {
|
||||
t.Fatalf("partial metadata confidence = %q, want medium", got)
|
||||
}
|
||||
|
||||
book = &parsedEbook{}
|
||||
if got := ebookIdentityConfidence(book); got != "low" {
|
||||
t.Fatalf("empty metadata confidence = %q, want low", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEbookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{
|
||||
Title: "Book", Year: 2024, Authors: []string{"Author"}, Description: "Overview", Publisher: "Publisher", Genres: []string{"Fiction"}, Language: "en",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveEbookMediaItem: %v", err)
|
||||
}
|
||||
if got == "" || len(writer.upserts) != 1 {
|
||||
t.Fatalf("contentID/upserts = %q/%d, want id and one upsert", got, len(writer.upserts))
|
||||
}
|
||||
item := writer.upserts[0]
|
||||
if item.Type != "ebook" || item.Title != "Book" || item.Year != 2024 || item.Overview != "Overview" || item.OriginalLanguage != "en" {
|
||||
t.Fatalf("upserted ebook item = %+v", item)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEbookMediaItemReusesRootScopedContentID(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{contentID: "ebook-root-id"}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{Title: "Book"})
|
||||
if err != nil {
|
||||
t.Fatalf("resolveEbookMediaItem: %v", err)
|
||||
}
|
||||
if got != "ebook-root-id" {
|
||||
t.Fatalf("contentID = %q, want root-scoped id", got)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveEbookExistingRootAppliesParsedMetadata(t *testing.T) {
|
||||
reader := &fakeFilesystemItemReader{items: []*models.MediaItem{{
|
||||
ContentID: "ebook-root-id",
|
||||
Type: "ebook",
|
||||
Title: "Old Title",
|
||||
Year: 2020,
|
||||
OriginalLanguage: "",
|
||||
}}}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
err := updateExistingEbookMediaItem(context.Background(), reader, writer, "ebook-root-id", &parsedEbook{
|
||||
Title: "New Title",
|
||||
Year: 2026,
|
||||
Publisher: "New Press",
|
||||
Genres: []string{"Fiction"},
|
||||
Language: "en",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("updateExistingEbookMediaItem: %v", err)
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
|
||||
}
|
||||
item := writer.upserts[0]
|
||||
if item.ContentID != "ebook-root-id" || item.Type != "ebook" || item.Title != "New Title" || item.Year != 2026 || item.OriginalLanguage != "en" {
|
||||
t.Fatalf("updated item = %+v", item)
|
||||
}
|
||||
if strings.Join(item.Studios, ",") != "New Press" || strings.Join(item.Genres, ",") != "Fiction" {
|
||||
t.Fatalf("updated item studios/genres = %+v/%+v", item.Studios, item.Genres)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEbookToMediaItemPreservesExistingYearWhenParsedYearMissing(t *testing.T) {
|
||||
item := &models.MediaItem{
|
||||
Type: "ebook",
|
||||
Title: "Existing",
|
||||
Year: 1999,
|
||||
}
|
||||
|
||||
applyEbookToMediaItem(item, &parsedEbook{Title: "Updated", Year: 0})
|
||||
|
||||
if item.Title != "Updated" {
|
||||
t.Fatalf("Title = %q, want Updated", item.Title)
|
||||
}
|
||||
if item.Year != 1999 {
|
||||
t.Fatalf("Year = %d, want preserved 1999", item.Year)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookPeopleCreditsEqualAuthorsOnly(t *testing.T) {
|
||||
existing := []models.ItemPerson{
|
||||
{Person: models.Person{Name: "Ada Writer"}, Kind: models.PersonKindAuthor, SortOrder: 0},
|
||||
{Person: models.Person{Name: "Ben Author"}, Kind: models.PersonKindAuthor, SortOrder: 1},
|
||||
{Person: models.Person{Name: "Manual Contributor"}, Kind: models.PersonKindWriter, SortOrder: 2},
|
||||
}
|
||||
desired := []ebookCredit{
|
||||
{Name: "Ada Writer", Kind: models.PersonKindAuthor},
|
||||
{Name: "Ben Author", Kind: models.PersonKindAuthor},
|
||||
}
|
||||
if !ebookPeopleCreditsEqual(existing, desired) {
|
||||
t.Fatal("expected matching author credits to compare equal while ignoring non-authors")
|
||||
}
|
||||
|
||||
existing[1] = models.ItemPerson{Person: models.Person{Name: "Other Author"}, Kind: models.PersonKindAuthor, SortOrder: 1}
|
||||
if ebookPeopleCreditsEqual(existing, desired) {
|
||||
t.Fatal("expected different author credit to make ebook author set differ")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookPeopleMergePreservesExistingNonAuthorCredits(t *testing.T) {
|
||||
existing := []models.ItemPerson{
|
||||
{Person: models.Person{ID: 10, Name: "Old Author"}, Kind: models.PersonKindAuthor, SortOrder: 0},
|
||||
{Person: models.Person{ID: 20, Name: "Manual Writer"}, Kind: models.PersonKindWriter, SortOrder: 1, Character: "essay"},
|
||||
{Person: models.Person{ID: 30, Name: "Manual Narrator"}, Kind: models.PersonKindNarrator, SortOrder: 2},
|
||||
}
|
||||
authors := []ebookResolvedAuthor{
|
||||
{ID: 40, Name: "New Author"},
|
||||
}
|
||||
|
||||
got := mergeEbookPeople(existing, authors)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("merged people len = %d, want 3: %+v", len(got), got)
|
||||
}
|
||||
if got[0].Person.ID != 20 || got[0].Kind != models.PersonKindWriter || got[0].Character != "essay" {
|
||||
t.Fatalf("first preserved non-author = %+v", got[0])
|
||||
}
|
||||
if got[1].Person.ID != 30 || got[1].Kind != models.PersonKindNarrator {
|
||||
t.Fatalf("second preserved non-author = %+v", got[1])
|
||||
}
|
||||
if got[2].Person.ID != 40 || got[2].Kind != models.PersonKindAuthor || got[2].SortOrder != 2 {
|
||||
t.Fatalf("new author credit = %+v", got[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookPeopleReplacePlanReturnsGetPeopleError(t *testing.T) {
|
||||
wantErr := errors.New("get people failed")
|
||||
_, err := ebookPeopleForReplace(nil, wantErr, []ebookResolvedAuthor{{ID: 40, Name: "New Author"}})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanEbookBuildMediaFileSetsCorePersistenceFields(t *testing.T) {
|
||||
modifiedAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
book := &parsedEbook{Format: "epub", Title: "Book", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
|
||||
|
||||
got := buildEbookMediaFile(&models.MediaFolder{ID: 44}, "content-1", "/library/Book.epub", 1234, modifiedAt, book)
|
||||
if got.ContentID != "content-1" || got.MediaFolderID != 44 {
|
||||
t.Fatalf("ids = %q/%d, want content-1/44", got.ContentID, got.MediaFolderID)
|
||||
}
|
||||
if got.BaseType != "ebook" || got.BaseTitle != "Book" || got.BaseYear != 2024 || got.Container != "epub" || got.ProbeSource != "local" {
|
||||
t.Fatalf("ebook file metadata = %+v", got)
|
||||
}
|
||||
if got.CanonicalRootPath != "/library/Book.epub" || got.ObservedRootPath != "/library/Book.epub" || got.FilePath != "/library/Book.epub" {
|
||||
t.Fatalf("ebook paths = canonical %q observed %q file %q", got.CanonicalRootPath, got.ObservedRootPath, got.FilePath)
|
||||
}
|
||||
if got.FileSize != 1234 || got.FileModifiedAt == nil || !got.FileModifiedAt.Equal(modifiedAt) {
|
||||
t.Fatalf("file facts = size %d modified %v", got.FileSize, got.FileModifiedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanEbookPersistenceSQLWritesLibraryMembershipAndISBNOnly(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
exec := &recordingEbookExecutor{}
|
||||
|
||||
if err := insertEbookLibraryMembership(ctx, exec, "content-1", 44); err != nil {
|
||||
t.Fatalf("insertEbookLibraryMembership: %v", err)
|
||||
}
|
||||
if err := insertEbookISBNProviderID(ctx, exec, "content-1", "9780306406157"); err != nil {
|
||||
t.Fatalf("insertEbookISBNProviderID: %v", err)
|
||||
}
|
||||
if len(exec.queries) != 2 {
|
||||
t.Fatalf("queries = %d, want 2", len(exec.queries))
|
||||
}
|
||||
if !strings.Contains(exec.queries[0], "media_item_libraries") || exec.args[0][0] != "content-1" || exec.args[0][1] != 44 {
|
||||
t.Fatalf("library membership write = query %q args %+v", exec.queries[0], exec.args[0])
|
||||
}
|
||||
if !strings.Contains(exec.queries[1], "media_item_provider_ids") || !strings.Contains(exec.queries[1], "'isbn'") || !strings.Contains(exec.queries[1], "'ebook'") {
|
||||
t.Fatalf("ISBN provider write query = %q", exec.queries[1])
|
||||
}
|
||||
if strings.Contains(exec.queries[1], "asin") || exec.args[1][0] != "content-1" || exec.args[1][1] != "9780306406157" {
|
||||
t.Fatalf("ISBN provider write args/query = query %q args %+v", exec.queries[1], exec.args[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanEbookFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "bad.epub"), []byte("not a real epub"), 0o644); err != nil {
|
||||
t.Fatalf("write fake ebook: %v", err)
|
||||
}
|
||||
|
||||
s := &Scanner{}
|
||||
err := s.ScanEbookFolder(context.Background(), &models.MediaFolder{ID: 44, Paths: []string{root}})
|
||||
if err == nil {
|
||||
t.Fatal("ScanEbookFolder returned nil, want aggregate failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "folder_id=44") {
|
||||
t.Fatalf("error = %q, want folder id", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanEbookFolderReturnsCanceledContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
s := &Scanner{}
|
||||
err := s.ScanEbookFolder(ctx, &models.MediaFolder{ID: 44, Paths: []string{root}})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ScanEbookFolder error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeTestEPUB(t *testing.T, identifiers []string) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "book.epub")
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create epub: %v", err)
|
||||
}
|
||||
zw := zip.NewWriter(file)
|
||||
add := func(name, body string) {
|
||||
t.Helper()
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatalf("create zip entry %s: %v", name, err)
|
||||
}
|
||||
if _, err := w.Write([]byte(body)); err != nil {
|
||||
t.Fatalf("write zip entry %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
add("META-INF/container.xml", `<?xml version="1.0"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>`)
|
||||
var identifierXML strings.Builder
|
||||
for _, identifier := range identifiers {
|
||||
identifierXML.WriteString(" <dc:identifier>")
|
||||
identifierXML.WriteString(identifier)
|
||||
identifierXML.WriteString("</dc:identifier>\n")
|
||||
}
|
||||
add("OPS/content.opf", `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
|
||||
<metadata>
|
||||
<dc:title>The Test Ebook</dc:title>
|
||||
<dc:creator>Ada Writer</dc:creator>
|
||||
<dc:creator>Ben Author</dc:creator>
|
||||
`+identifierXML.String()+`
|
||||
<dc:publisher>Silo Press</dc:publisher>
|
||||
<dc:date>2024-03-10</dc:date>
|
||||
<dc:language>en</dc:language>
|
||||
<dc:subject>Fiction</dc:subject>
|
||||
<dc:subject>Adventure</dc:subject>
|
||||
<dc:description>Back cover copy</dc:description>
|
||||
<meta name="calibre:series" content="Test Series"/>
|
||||
<meta name="calibre:series_index" content="2"/>
|
||||
</metadata>
|
||||
</package>`)
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("close zip: %v", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatalf("close epub: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -241,15 +241,27 @@ func (s *Scanner) ScanFolder(ctx context.Context, folder *models.MediaFolder) (*
|
||||
return &ScanResult{}, nil
|
||||
}
|
||||
|
||||
if isEbookLibraryType(folder.Type) {
|
||||
if err := s.ScanEbookFolder(watchCtx, folder); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ScanResult{}, nil
|
||||
}
|
||||
|
||||
return s.scanPaths(watchCtx, folder, folder.Paths, folder.Paths, true)
|
||||
}
|
||||
|
||||
var errEbookScanningNotImplemented = errors.New("ebook scanning is not implemented")
|
||||
|
||||
// ScanSubtree walks a single subtree within a media folder and reconciles only
|
||||
// files that live beneath that subtree.
|
||||
func (s *Scanner) ScanSubtree(ctx context.Context, folder *models.MediaFolder, subtreePath string) (*ScanResult, error) {
|
||||
cleanSubtree := filepath.Clean(subtreePath)
|
||||
watchCtx, stopWatch := s.watchFolderContext(ctx, folder.ID)
|
||||
defer stopWatch()
|
||||
if isEbookLibraryType(folder.Type) {
|
||||
return nil, errEbookScanningNotImplemented
|
||||
}
|
||||
return s.scanPaths(watchCtx, folder, []string{cleanSubtree}, []string{cleanSubtree}, false)
|
||||
}
|
||||
|
||||
@@ -278,6 +290,15 @@ func isPodcastLibraryType(libraryType string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func isEbookLibraryType(libraryType string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(libraryType)) {
|
||||
case "ebook", "ebooks":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// walkMode tells walkLogicalTree which file extensions to surface and
|
||||
// which library-specific filename heuristics (sample/extra skipping)
|
||||
// to apply.
|
||||
@@ -288,6 +309,7 @@ const (
|
||||
walkModeMovie // movie library: video extensions + sample/extra skipping
|
||||
walkModeAudiobook // audiobook library: audio extensions, no skipping
|
||||
walkModePodcast // podcast library: audio extensions, no skipping
|
||||
walkModeEbook // ebook library: ebook extensions, no skipping
|
||||
)
|
||||
|
||||
// walkModeFor derives a walkMode from a media_folders.type string.
|
||||
@@ -301,6 +323,8 @@ func walkModeFor(folderType string) walkMode {
|
||||
return walkModeAudiobook
|
||||
case isPodcastLibraryType(folderType):
|
||||
return walkModePodcast
|
||||
case isEbookLibraryType(folderType):
|
||||
return walkModeEbook
|
||||
default:
|
||||
return walkModeVideo
|
||||
}
|
||||
@@ -312,6 +336,8 @@ func (m walkMode) acceptsExt(ext string) bool {
|
||||
switch m {
|
||||
case walkModeAudiobook, walkModePodcast:
|
||||
return audioExtensions[ext]
|
||||
case walkModeEbook:
|
||||
return ebookExtensions[ext]
|
||||
default:
|
||||
return videoExtensions[ext]
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -200,3 +201,82 @@ func TestIsPodcastLibraryType(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEbookLibraryType(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"ebooks", true},
|
||||
{"ebook", true},
|
||||
{"Ebook", true},
|
||||
{" EBOOKS ", true},
|
||||
{"audiobooks", false},
|
||||
{"movies", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := isEbookLibraryType(tc.in); got != tc.want {
|
||||
t.Errorf("isEbookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkModeForEbookLibraryTypes(t *testing.T) {
|
||||
for _, libraryType := range []string{"ebook", "ebooks", " EBOOKS "} {
|
||||
if got := walkModeFor(libraryType); got != walkModeEbook {
|
||||
t.Fatalf("walkModeFor(%q) = %v, want %v", libraryType, got, walkModeEbook)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkModeEbookAcceptsEbookExtensionsOnly(t *testing.T) {
|
||||
if !walkModeEbook.acceptsExt(".epub") {
|
||||
t.Fatal("walkModeEbook should accept .epub")
|
||||
}
|
||||
if !walkModeEbook.acceptsExt(".pdf") {
|
||||
t.Fatal("walkModeEbook should accept .pdf")
|
||||
}
|
||||
if walkModeEbook.acceptsExt(".mp4") {
|
||||
t.Fatal("walkModeEbook should reject video extensions")
|
||||
}
|
||||
if walkModeEbook.acceptsExt(".mp3") {
|
||||
t.Fatal("walkModeEbook should reject audio extensions")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanFolderEbookLibraryRoutesToEbookScanner(t *testing.T) {
|
||||
scanner := &Scanner{}
|
||||
result, err := scanner.ScanFolder(context.Background(), &models.MediaFolder{
|
||||
ID: 0,
|
||||
Type: " ebooks ",
|
||||
Paths: []string{t.TempDir()},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScanFolder ebook error = %v, want nil", err)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("ScanFolder ebook result = nil, want empty result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanSubtreeEbookLibraryReturnsNotImplemented(t *testing.T) {
|
||||
subtree := t.TempDir()
|
||||
ebookPath := filepath.Join(subtree, "Book.epub")
|
||||
if err := os.WriteFile(ebookPath, []byte("not a real epub"), 0o644); err != nil {
|
||||
t.Fatalf("write ebook fixture: %v", err)
|
||||
}
|
||||
|
||||
scanner := &Scanner{}
|
||||
result, err := scanner.ScanSubtree(context.Background(), &models.MediaFolder{
|
||||
ID: 0,
|
||||
Type: "ebook",
|
||||
Paths: []string{subtree},
|
||||
}, subtree)
|
||||
if !errors.Is(err, errEbookScanningNotImplemented) {
|
||||
t.Fatalf("ScanSubtree ebook error = %v, want %v", err, errEbookScanningNotImplemented)
|
||||
}
|
||||
if result != nil {
|
||||
t.Fatalf("ScanSubtree ebook result = %+v, want nil", result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE ebook_series (
|
||||
content_id TEXT PRIMARY KEY REFERENCES media_items(content_id) ON DELETE CASCADE,
|
||||
series_name TEXT NOT NULL,
|
||||
series_index NUMERIC,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX ebook_series_name_lower
|
||||
ON ebook_series (LOWER(series_name), series_index NULLS LAST);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS ebook_series;
|
||||
-- +goose StatementEnd
|
||||
Reference in New Issue
Block a user