Compare commits

...
8 changed files with 2344 additions and 0 deletions
@@ -0,0 +1,466 @@
# Ebooks Like Audiobooks Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Add ebooks as a core silo media type by mirroring the audiobook scanner architecture in `main`. Metadata enrichment is documented below as follow-up PR work, not part of the foundation PR.
**Architecture:** The foundation PR scans ebooks through core `internal/scanner` code. Later PRs add a core `internal/ebooks.Enricher` task that resolves plugin metadata providers with `content_level = 'ebook'`. The first-party `silo-plugin-ebook-metadata` provides lookup data; there is no ebook scanner plugin and no ebook details repository/table.
**Tech Stack:** Go, PostgreSQL Goose migrations, Silo scanner/catalog repositories, Silo metadata plugin chain, taskmanager scheduled tasks.
---
## File Structure
- `internal/scanner/ebook.go`: local ebook parser and format support.
- `internal/scanner/ebook_scan.go`: ebook scan flow matching `audiobook_scan.go`.
- `internal/scanner/scanner.go`: route `ebook(s)` libraries to the ebook scanner and add ebook walk mode.
- `migrations/sql/20260608000100_ebook_series.sql`: `ebook_series` table equivalent to `audiobook_series`.
- Follow-up PR: `internal/ebooks/enrichment.go`: ebook metadata sweep equivalent to `internal/audiobooks/enrichment.go`.
- Follow-up PR: `internal/taskmanager/tasks/sync_ebook_metadata.go`: scheduled/manual metadata task equivalent to `sync_audiobook_metadata.go`.
- Follow-up PR: `cmd/silo/main.go`: wire `ebooks.NewEnricher` next to `audiobooks.NewEnricher`.
- Tests live beside changed files and should pin “authors only, ISBN only, content_level ebook.”
### Task 1: Scanner Routing And Local Parser
**Files:**
- Modify: `internal/scanner/scanner.go`
- Modify: `internal/scanner/scanner_test.go`
- Create/modify: `internal/scanner/ebook.go`
- Create/modify: `internal/scanner/ebook_test.go`
- [ ] **Step 1: Write or keep failing scanner/parser tests**
Ensure these tests exist:
```go
func TestIsEbookLibraryType(t *testing.T) {
cases := []struct {
in string
want bool
}{
{"ebooks", true},
{"ebook", true},
{"Ebook", true},
{" EBOOKS ", true},
{"audiobooks", false},
{"movies", false},
{"", false},
}
for _, tc := range cases {
if got := isEbookLibraryType(tc.in); got != tc.want {
t.Errorf("isEbookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
}
}
}
func TestWalkModeForEbookLibraryTypes(t *testing.T) {
for _, libraryType := range []string{"ebook", "ebooks", " EBOOKS "} {
if got := walkModeFor(libraryType); got != walkModeEbook {
t.Fatalf("walkModeFor(%q) = %v, want walkModeEbook", libraryType, got)
}
}
}
func TestSupportsEbookFile(t *testing.T) {
cases := []struct {
path string
want bool
}{
{"book.epub", true},
{"book.PDF", true},
{"book.azw3", true},
{"book.mp3", false},
{"movie.mkv", false},
}
for _, tc := range cases {
if got := SupportsEbookFile(tc.path); got != tc.want {
t.Errorf("SupportsEbookFile(%q) = %v, want %v", tc.path, got, tc.want)
}
}
}
```
- [ ] **Step 2: Run scanner/parser red test**
Run:
```bash
go test ./internal/scanner -run 'Test(IsEbook|WalkModeForEbook|WalkModeEbook|SupportsEbook|ParseEbook)' -count=1
```
Expected before implementation: fail for missing ebook mode/parser functions. Expected after existing parser work is kept: pass.
- [ ] **Step 3: Implement minimal parser and walk mode**
Implement:
```go
var ebookExtensions = map[string]bool{
".epub": true,
".pdf": true,
".mobi": true,
".azw": true,
".azw3": true,
".fb2": true,
}
func SupportsEbookFile(filePath string) bool {
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
}
func isEbookLibraryType(libraryType string) bool {
switch strings.ToLower(strings.TrimSpace(libraryType)) {
case "ebook", "ebooks":
return true
default:
return false
}
}
```
Add `walkModeEbook` and make `walkMode.acceptsExt` call `SupportsEbookFile`.
- [ ] **Step 4: Run scanner/parser green test**
Run the same command as Step 2. Expected: `ok github.com/Silo-Server/silo-server/internal/scanner`.
- [ ] **Step 5: Commit Task 1**
```bash
git add internal/scanner/scanner.go internal/scanner/scanner_test.go internal/scanner/ebook.go internal/scanner/ebook_test.go
git commit -m "feat: add ebook scanner parser foundation"
```
### Task 2: Ebook Core Scan Persistence
**Files:**
- Create: `internal/scanner/ebook_scan.go`
- Modify: `internal/scanner/ebook_test.go`
- Modify: `internal/scanner/scanner.go`
- [ ] **Step 1: Write failing tests for audiobook-shaped ebook helpers**
Add tests for:
```go
func TestEbookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
book := &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
if got := ebookIdentityConfidence(book); got != "high" {
t.Fatalf("complete metadata confidence = %q, want high", got)
}
book = &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024}
if got := ebookIdentityConfidence(book); got != "medium" {
t.Fatalf("partial metadata confidence = %q, want medium", got)
}
book = &parsedEbook{}
if got := ebookIdentityConfidence(book); got != "low" {
t.Fatalf("empty metadata confidence = %q, want low", got)
}
}
func TestResolveEbookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
finder := &fakeRootContentFinder{}
writer := &fakeFilesystemItemWriter{}
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{
Title: "Book", Year: 2024, Authors: []string{"Author"}, Description: "Overview", Publisher: "Publisher", Genres: []string{"Fiction"}, Language: "en",
})
if err != nil {
t.Fatalf("resolveEbookMediaItem: %v", err)
}
if got == "" || len(writer.upserts) != 1 {
t.Fatalf("contentID/upserts = %q/%d, want id and one upsert", got, len(writer.upserts))
}
item := writer.upserts[0]
if item.Type != "ebook" || item.Title != "Book" || item.Year != 2024 || item.Overview != "Overview" || item.OriginalLanguage != "en" {
t.Fatalf("upserted ebook item = %+v", item)
}
}
```
- [ ] **Step 2: Run red helper tests**
Run:
```bash
go test ./internal/scanner -run 'Test(EbookIdentity|ResolveEbook|EbookPeople)' -count=1
```
Expected before implementation: fail with undefined ebook scan helper functions.
- [ ] **Step 3: Implement ebook scan helpers mirroring audiobook names**
Create `internal/scanner/ebook_scan.go` with functions shaped after `audiobook_scan.go`:
```go
func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolder) error
func (s *Scanner) reconcileEbookFile(ctx context.Context, folder *models.MediaFolder, filePath string, skipped *int64) error
func (s *Scanner) ebookFileShouldSkip(ctx context.Context, folder *models.MediaFolder, filePath string, size int64, modifiedAt time.Time) (bool, error)
func (s *Scanner) upsertEbookMediaItem(ctx context.Context, folderID int, filePath string, book *parsedEbook) (string, error)
func resolveEbookMediaItem(ctx context.Context, rootFinder filesystemRootContentFinder, itemWriter filesystemMediaItemWriter, folderID int, filePath string, book *parsedEbook) (string, error)
func createEbookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedEbook, cleanTitle string) (string, error)
func applyEbookToMediaItem(item *models.MediaItem, book *parsedEbook)
func (s *Scanner) upsertEbookMediaFile(ctx context.Context, folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) error
func (s *Scanner) upsertEbookPeople(ctx context.Context, contentID string, book *parsedEbook) error
func (s *Scanner) upsertEbookSeries(ctx context.Context, contentID string, book *parsedEbook) error
func ebookIdentityConfidence(book *parsedEbook) string
```
Behavior:
- `ScanEbookFolder` walks `folder.Paths`, collects files where `SupportsEbookFile` is true, processes in a worker pool like `ScanAudiobookFolder`, logs `ebook scan: starting/progress/completed/indexed`.
- `upsertEbookMediaItem` reuses `FindContentIDByRootPath(ctx, folderID, filePath, "ebook")`, then finds by existing `media_files.file_path`, then creates a new `media_items` row.
- `applyEbookToMediaItem` sets type/title/year/overview/publisher-as-studio/genres/release date/original language.
- `upsertEbookMediaFile` writes one `media_files` row with `CanonicalRootPath`, `ObservedRootPath`, `ContentGroupKey`, `FilePath` all tied to the ebook file path or content ID in the same style as audiobook file persistence.
- `upsertEbookPeople` writes author credits only with `models.PersonKindAuthor`; it never writes `models.PersonKindNarrator`.
- `reconcileEbookFile` writes ISBN provider ID with:
```sql
INSERT INTO media_item_provider_ids (content_id, provider, provider_id, item_type)
VALUES ($1, 'isbn', $2, 'ebook')
ON CONFLICT DO NOTHING
```
- [ ] **Step 4: Route ebook libraries in `ScanFolder`**
Add this block beside audiobook/podcast routing:
```go
if isEbookLibraryType(folder.Type) {
if err := s.ScanEbookFolder(watchCtx, folder); err != nil {
return nil, err
}
return &ScanResult{}, nil
}
```
- [ ] **Step 5: Run green helper tests**
Run:
```bash
go test ./internal/scanner -run 'Test(EbookIdentity|ResolveEbook|EbookPeople|ScanEbook)' -count=1
```
Expected: scanner tests pass.
- [ ] **Step 6: Commit Task 2**
```bash
git add internal/scanner/scanner.go internal/scanner/ebook_scan.go internal/scanner/ebook_test.go
git commit -m "feat: scan ebook libraries in core"
```
### Task 3: Ebook Series Migration
**Files:**
- Create: `migrations/sql/20260608000100_ebook_series.sql`
- Modify: `internal/scanner/ebook_scan.go`
- [ ] **Step 1: Add migration mirroring `audiobook_series`**
Create:
```sql
-- +goose Up
-- +goose StatementBegin
CREATE TABLE ebook_series (
content_id TEXT PRIMARY KEY REFERENCES media_items(content_id) ON DELETE CASCADE,
series_name TEXT NOT NULL,
series_index NUMERIC,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX ebook_series_name_lower
ON ebook_series (LOWER(series_name), series_index NULLS LAST);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS ebook_series;
-- +goose StatementEnd
```
- [ ] **Step 2: Add scanner series upsert SQL**
Use the same write/delete/skip-identical shape as `upsertAudiobookSeries`, with table name `ebook_series` and `book.SeriesIndex` parsed through existing `parseSeriesIndex`.
- [ ] **Step 3: Run migration and scanner tests**
Run:
```bash
go test ./internal/scanner -run 'Test(Ebook|ResolveEbook)' -count=1
```
Expected: pass.
- [ ] **Step 4: Commit Task 3**
```bash
git add migrations/sql/20260608000100_ebook_series.sql internal/scanner/ebook_scan.go
git commit -m "feat: persist ebook series membership"
```
### Follow-up PR Task 4: Ebook Metadata Enricher
**Files:**
- Create: `internal/ebooks/enrichment.go`
- Create: `internal/ebooks/enrichment_test.go`
- [ ] **Step 1: Copy audiobook enricher structure with ebook domain changes**
Create `internal/ebooks/enrichment.go` from `internal/audiobooks/enrichment.go` and change:
- package `ebooks`
- log prefix `ebook enrichment`
- default env var `SILO_EBOOK_ENRICH_WORKERS`
- batch query `WHERE mi.type = 'ebook'`
- chain resolution `metadata.ResolveChain(ctx, item.FolderID, "ebook", e.chainRepo, e.resolver)`
- search query `ContentType: "ebook"`
- metadata request `ContentType: "ebook"`
- people persistence filters to author credits only; discard narrator credits if a plugin returns them
- no local ffmpeg audiobook cover fallback
- [ ] **Step 2: Add tests for chain content level and authors-only people**
Add tests that instantiate an `Enricher` with fake provider behavior where possible, or unit-test narrow helpers:
```go
func TestEbookEnricherContentType(t *testing.T) {
if got := ebookContentType(); got != "ebook" {
t.Fatalf("ebookContentType() = %q, want ebook", got)
}
}
func TestFilterEbookPeopleKeepsAuthorsOnly(t *testing.T) {
in := []models.ItemPerson{
{Person: models.Person{Name: "Author"}, Kind: models.PersonKindAuthor},
{Person: models.Person{Name: "Narrator"}, Kind: models.PersonKindNarrator},
}
got := filterEbookPeople(in)
if len(got) != 1 || got[0].Kind != models.PersonKindAuthor || got[0].Person.Name != "Author" {
t.Fatalf("filterEbookPeople() = %+v, want author only", got)
}
}
```
- [ ] **Step 3: Run red/green ebook enricher tests**
Run:
```bash
go test ./internal/ebooks -count=1
```
Expected after implementation: pass.
- [ ] **Step 4: Commit Task 4**
```bash
git add internal/ebooks/enrichment.go internal/ebooks/enrichment_test.go
git commit -m "feat: add ebook metadata enricher"
```
### Follow-up PR Task 5: Ebook Metadata Task And Main Wiring
**Files:**
- Create: `internal/taskmanager/tasks/sync_ebook_metadata.go`
- Modify: `cmd/silo/main.go`
- [ ] **Step 1: Add sync task matching audiobook task**
Create task:
```go
type SyncEbookMetadataTask struct {
enricher *ebooks.Enricher
}
func NewSyncEbookMetadataTask(enricher *ebooks.Enricher) *SyncEbookMetadataTask {
return &SyncEbookMetadataTask{enricher: enricher}
}
func (t *SyncEbookMetadataTask) Key() string { return "sync_ebook_metadata" }
func (t *SyncEbookMetadataTask) Name() string { return "Sync Ebook Metadata" }
func (t *SyncEbookMetadataTask) Description() string {
return "Fetches metadata (cover art, overview, authors) for ebooks that have not yet been enriched"
}
func (t *SyncEbookMetadataTask) Category() taskmanager.TaskCategory {
return taskmanager.TaskCategoryMetadata
}
func (t *SyncEbookMetadataTask) IsHidden() bool { return false }
func (t *SyncEbookMetadataTask) DefaultTriggers() []taskmanager.TriggerConfig {
return []taskmanager.TriggerConfig{{Type: taskmanager.TriggerTypeInterval, IntervalMs: 5 * 60 * 1000}}
}
```
- [ ] **Step 2: Wire `ebooks.NewEnricher` beside audiobook enricher**
In `cmd/silo/main.go`:
- add import `github.com/Silo-Server/silo-server/internal/ebooks`
- add `var ebookEnricher *ebooks.Enricher`
- construct it with the same dependencies as `audiobooks.NewEnricher`
- set image cacher if object storage is available and the ebook enricher supports remote image caching
- register `tasks.NewSyncEbookMetadataTask(ebookEnricher)` beside `NewSyncAudiobookMetadataTask`
- [ ] **Step 3: Run task/main build tests**
Run:
```bash
go test ./internal/taskmanager/tasks ./internal/ebooks -count=1
go test ./cmd/silo -run TestNonExistent -count=1
```
Expected: packages compile and tests pass.
- [ ] **Step 4: Commit Task 5**
```bash
git add cmd/silo/main.go internal/taskmanager/tasks/sync_ebook_metadata.go
git commit -m "feat: wire ebook metadata sync task"
```
### Task 6: Verification And PR Cleanup
**Files:**
- Review all modified files.
- [ ] **Step 1: Run focused Go tests**
```bash
go test ./internal/scanner ./internal/ebooks ./internal/taskmanager/tasks -count=1
```
Expected: pass.
- [ ] **Step 2: Run broader compile/test set**
```bash
go test ./internal/catalog ./internal/metadata ./cmd/silo -count=1
```
Expected: pass, or record unrelated baseline failures with exact package/error.
- [ ] **Step 3: Check no wrong architecture remains**
Run:
```bash
rg -n "ebook_details|silo-plugin-ebooks|PersonKindNarrator|asin" internal/scanner internal/ebooks docs/superpowers/specs docs/superpowers/plans
```
Expected:
- no `ebook_details`
- no scanner dependency on `silo-plugin-ebooks`
- no ebook code writing `PersonKindNarrator`
- no ebook code writing ASIN
- [ ] **Step 4: Commit final cleanup if needed**
```bash
git status --short
git add <only ebook-related cleanup files>
git commit -m "test: verify ebook audiobook-parity path"
```
Only commit if Step 1-3 caused file changes.
@@ -0,0 +1,102 @@
# Ebooks Like Audiobooks Design
## Goal
Add ebooks to silo by matching the audiobook architecture in `main` as closely
as possible. Ebooks are a core media type in `silo-server`; only metadata
provider lookup lives in a plugin.
## Non-Goals
- Do not add or keep a scanner/runtime plugin named `silo-plugin-ebooks`.
- Do not treat ebooks as an audiobook sub-feature.
- Do not add ebook narrators.
- Do not add ebook ASIN handling.
- Do not introduce an ebook details repository/table unless the audiobook path
gains the same shape first.
## Audiobook Pattern In Main
Audiobooks have three cooperating pieces:
1. Core scan:
- `internal/scanner/scanner.go` routes `audiobook` and `audiobooks` library
types to `ScanAudiobookFolder`.
- `internal/scanner/audiobook_scan.go` writes `media_items`,
`media_files`, `item_people`, `audiobook_series`,
`media_item_libraries`, and durable provider IDs.
2. Core metadata enrichment:
- `cmd/silo/main.go` wires `audiobooks.NewEnricher`.
- `internal/taskmanager/tasks/sync_audiobook_metadata.go` registers the
periodic/manual task.
- `internal/audiobooks/enrichment.go` sweeps unenriched `audiobook` items,
resolves the configured metadata provider chain at
`content_level = 'audiobook'`, calls `Search` then `GetMetadata`, and
persists provider IDs, poster/overview/scalar metadata, and people.
3. First-party metadata plugin:
- `silo-plugin-audiobook-metadata` exposes `metadata_provider.v1` with an
audiobook capability and provider implementations.
## Ebook Equivalent Across PRs
Ebooks should mirror that shape:
1. Core scan:
- `internal/scanner/scanner.go` routes `ebook` and `ebooks` libraries to an
ebook scan path.
- `internal/scanner/ebook.go` parses local ebook metadata from supported
files.
- `internal/scanner/ebook_scan.go` writes:
- `media_items.type = 'ebook'`
- `media_files.base_type = 'ebook'`
- `item_people` author credits only
- `ebook_series`
- `media_item_libraries`
- `media_item_provider_ids` using provider `isbn` when an ISBN is parsed
2. Follow-up core metadata enrichment:
- Add `internal/ebooks/enrichment.go`, shaped after
`internal/audiobooks/enrichment.go`.
- Add `internal/taskmanager/tasks/sync_ebook_metadata.go`, shaped after
`sync_audiobook_metadata.go`.
- Wire `ebooks.NewEnricher` in `cmd/silo/main.go` next to the audiobook
enricher.
- Resolve provider chains at `content_level = 'ebook'`.
- Search and fetch metadata with `ContentType = "ebook"`.
- Persist provider IDs, poster/overview/scalar metadata, and author people.
3. First-party metadata plugin:
- Use `silo-plugin-ebook-metadata` as the first-party ebook metadata
provider plugin.
- The core app should rely on that plugin for provider-specific metadata
lookups, just as audiobooks rely on `silo-plugin-audiobook-metadata`.
## Ebook Domain Rules
- Ebooks have ISBNs.
- Ebooks have authors.
- Ebooks do not have narrators.
- Ebooks do not have ASINs.
- Ebook metadata plugins may return provider IDs, but core scanner-derived ISBN
should be stored under provider `isbn`.
- Local parsing may read embedded ebook metadata from EPUB/FB2 and format/file
facts from other supported ebook formats.
## First PR Scope
The first implementation PR should establish the mirrored foundation:
- ebook library type and walk mode
- local ebook parser
- ebook scan routing and persistence
- `ebook_series` migration
- tests proving ebook authors-only behavior, ISBN provider ID behavior, and
scanner-owned ebook series behavior
Deferred follow-up PR scope:
- ebook metadata enrichment package/task/wiring, analogous to
`internal/audiobooks.Enricher`
- plugin chain resolution at `content_level = 'ebook'`
UI/detail page, ebook reader experience, ABS ebook compatibility endpoints, and
advanced provider-specific plugin improvements can be separate PRs unless they
are required to prove the core path works.
+377
View File
@@ -0,0 +1,377 @@
package scanner
import (
"archive/zip"
"encoding/xml"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const maxEPUBMetadataEntrySize = 8 * 1024 * 1024
var ebookExtensions = map[string]bool{
".epub": true,
".pdf": true,
".mobi": true,
".azw": true,
".azw3": true,
".fb2": true,
}
type parsedEbook struct {
Format string
Title string
Authors []string
Description string
Publisher string
PublishedAt time.Time
Year int
Language string
ISBN string
Series string
SeriesIndex string
Genres []string
PageCount int
Cover *parsedEbookCover
}
type parsedEbookCover struct {
ContentType string
Bytes []byte
}
func SupportsEbookFile(filePath string) bool {
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
}
func parseEbookFile(path string) (book parsedEbook, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic parsing ebook %s: %v", path, r)
}
}()
switch strings.ToLower(filepath.Ext(path)) {
case ".epub":
book, err = parseEbookEPUB(path)
case ".fb2":
book, err = parseEbookFB2(path)
case ".pdf", ".mobi", ".azw", ".azw3":
book = parsedEbook{Format: strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")}
default:
err = fmt.Errorf("unsupported ebook format: %s", filepath.Ext(path))
}
book.sanitize()
return book, err
}
func (b *parsedEbook) sanitize() {
b.Format = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(b.Format)), ".")
b.Title = strings.TrimSpace(b.Title)
b.Description = strings.TrimSpace(b.Description)
b.Publisher = strings.TrimSpace(b.Publisher)
b.Language = strings.TrimSpace(b.Language)
b.ISBN = normalizeEbookISBN(b.ISBN)
b.Series = strings.TrimSpace(b.Series)
b.SeriesIndex = strings.TrimSpace(b.SeriesIndex)
b.Authors = uniqueTrimmedStrings(b.Authors)
b.Genres = uniqueTrimmedStrings(b.Genres)
if b.PageCount < 0 {
b.PageCount = 0
}
if b.Year == 0 && !b.PublishedAt.IsZero() {
b.Year = b.PublishedAt.Year()
}
}
func uniqueTrimmedStrings(values []string) []string {
seen := map[string]struct{}{}
var out []string
for _, value := range values {
trimmed := strings.TrimSpace(value)
key := strings.ToLower(trimmed)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, trimmed)
}
return out
}
func normalizeEbookISBN(value string) string {
value = strings.ToUpper(strings.TrimSpace(value))
for _, prefix := range []string{"ISBN-13", "ISBN-10", "ISBN"} {
if strings.HasPrefix(value, prefix) {
value = strings.TrimSpace(strings.TrimPrefix(value, prefix))
value = strings.TrimLeft(value, ": ")
break
}
}
var out strings.Builder
for _, r := range value {
if r >= '0' && r <= '9' {
out.WriteRune(r)
continue
}
if r == 'X' {
out.WriteRune(r)
}
}
candidate := out.String()
switch len(candidate) {
case 10:
if validISBN10(candidate) {
return candidate
}
case 13:
if validISBN13(candidate) {
return candidate
}
}
return ""
}
func validISBN10(candidate string) bool {
if len(candidate) != 10 {
return false
}
sum := 0
for i, r := range candidate {
value := 0
switch {
case r >= '0' && r <= '9':
value = int(r - '0')
case r == 'X' && i == 9:
value = 10
default:
return false
}
sum += value * (10 - i)
}
return sum%11 == 0
}
func validISBN13(candidate string) bool {
if len(candidate) != 13 {
return false
}
sum := 0
for i, r := range candidate {
if r < '0' || r > '9' {
return false
}
value := int(r - '0')
if i%2 == 1 {
value *= 3
}
sum += value
}
return sum%10 == 0
}
func parseEbookEPUB(path string) (parsedEbook, error) {
book := parsedEbook{Format: "epub"}
reader, err := zip.OpenReader(path)
if err != nil {
return book, err
}
defer reader.Close()
container, err := readEPUBZipEntry(&reader.Reader, "META-INF/container.xml")
if err != nil {
return book, err
}
opfPath, err := epubOPFPath(container)
if err != nil {
return book, err
}
opf, err := readEPUBZipEntry(&reader.Reader, opfPath)
if err != nil {
return book, err
}
if err := parseEPUBOPFMetadata(opf, &book); err != nil {
return book, err
}
return book, nil
}
func parseEbookFB2(path string) (parsedEbook, error) {
book := parsedEbook{Format: "fb2"}
file, err := os.Open(path)
if err != nil {
return book, err
}
defer file.Close()
var fb2 struct {
Description struct {
TitleInfo struct {
Genres []string `xml:"genre"`
Authors []struct {
FirstName string `xml:"first-name"`
MiddleName string `xml:"middle-name"`
LastName string `xml:"last-name"`
Nickname string `xml:"nickname"`
} `xml:"author"`
BookTitle string `xml:"book-title"`
Lang string `xml:"lang"`
Date struct {
Value string `xml:"value,attr"`
Text string `xml:",chardata"`
} `xml:"date"`
Sequences []struct {
Name string `xml:"name,attr"`
Number string `xml:"number,attr"`
} `xml:"sequence"`
} `xml:"title-info"`
PublishInfo struct {
ISBN string `xml:"isbn"`
Publisher string `xml:"publisher"`
Year string `xml:"year"`
} `xml:"publish-info"`
} `xml:"description"`
}
if err := xml.NewDecoder(file).Decode(&fb2); err != nil {
return book, err
}
book.Title = fb2.Description.TitleInfo.BookTitle
book.Language = fb2.Description.TitleInfo.Lang
book.Genres = fb2.Description.TitleInfo.Genres
if t, ok := parseEbookDate(firstNonEmpty(fb2.Description.TitleInfo.Date.Value, fb2.Description.TitleInfo.Date.Text)); ok {
book.PublishedAt = t
}
for _, author := range fb2.Description.TitleInfo.Authors {
name := strings.Join(uniqueTrimmedStrings([]string{
author.FirstName,
author.MiddleName,
author.LastName,
}), " ")
if name == "" {
name = author.Nickname
}
book.Authors = append(book.Authors, name)
}
if len(fb2.Description.TitleInfo.Sequences) > 0 {
book.Series = fb2.Description.TitleInfo.Sequences[0].Name
book.SeriesIndex = fb2.Description.TitleInfo.Sequences[0].Number
}
book.ISBN = fb2.Description.PublishInfo.ISBN
book.Publisher = fb2.Description.PublishInfo.Publisher
if year, err := strconv.Atoi(strings.TrimSpace(fb2.Description.PublishInfo.Year)); err == nil {
book.Year = year
}
return book, nil
}
func readEPUBZipEntry(reader *zip.Reader, name string) ([]byte, error) {
for _, file := range reader.File {
if file.Name != name {
continue
}
entry, err := file.Open()
if err != nil {
return nil, err
}
defer entry.Close()
limited := io.LimitReader(entry, maxEPUBMetadataEntrySize+1)
data, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if len(data) > maxEPUBMetadataEntrySize {
return nil, fmt.Errorf("epub entry too large: %s", name)
}
return data, nil
}
return nil, fmt.Errorf("epub entry not found: %s", name)
}
func epubOPFPath(container []byte) (string, error) {
var parsed struct {
Rootfiles []struct {
FullPath string `xml:"full-path,attr"`
} `xml:"rootfiles>rootfile"`
}
if err := xml.Unmarshal(container, &parsed); err != nil {
return "", err
}
for _, rootfile := range parsed.Rootfiles {
if strings.TrimSpace(rootfile.FullPath) != "" {
return rootfile.FullPath, nil
}
}
return "", fmt.Errorf("epub container has no rootfile")
}
func parseEPUBOPFMetadata(opf []byte, book *parsedEbook) error {
var parsed struct {
Metadata struct {
Titles []string `xml:"title"`
Creators []string `xml:"creator"`
Identifiers []string `xml:"identifier"`
Publisher string `xml:"publisher"`
Dates []string `xml:"date"`
Language string `xml:"language"`
Subjects []string `xml:"subject"`
Descriptions []string `xml:"description"`
Meta []struct {
Name string `xml:"name,attr"`
Property string `xml:"property,attr"`
Content string `xml:"content,attr"`
Value string `xml:",chardata"`
} `xml:"meta"`
} `xml:"metadata"`
}
if err := xml.Unmarshal(opf, &parsed); err != nil {
return err
}
book.Title = firstNonEmpty(parsed.Metadata.Titles...)
book.Authors = append(book.Authors, parsed.Metadata.Creators...)
book.Publisher = parsed.Metadata.Publisher
book.Language = parsed.Metadata.Language
book.Genres = append(book.Genres, parsed.Metadata.Subjects...)
book.Description = firstNonEmpty(parsed.Metadata.Descriptions...)
for _, identifier := range parsed.Metadata.Identifiers {
if isbn := normalizeEbookISBN(identifier); isbn != "" {
book.ISBN = isbn
break
}
}
for _, date := range parsed.Metadata.Dates {
if t, ok := parseEbookDate(date); ok {
book.PublishedAt = t
book.Year = t.Year()
break
}
}
for _, meta := range parsed.Metadata.Meta {
name := strings.ToLower(strings.TrimSpace(firstNonEmpty(meta.Name, meta.Property)))
value := strings.TrimSpace(firstNonEmpty(meta.Content, meta.Value))
switch name {
case "calibre:series", "belongs-to-collection":
book.Series = value
case "calibre:series_index", "group-position":
book.SeriesIndex = value
}
}
return nil
}
func parseEbookDate(value string) (time.Time, bool) {
value = strings.TrimSpace(value)
for _, layout := range []string{"2006-01-02", "2006-01", "2006"} {
if t, err := time.Parse(layout, value); err == nil {
return t, true
}
}
return time.Time{}, false
}
+686
View File
@@ -0,0 +1,686 @@
package scanner
import (
"context"
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Silo-Server/silo-server/internal/idgen"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/titleutil"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type ebookSQLExecutor interface {
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
}
type filesystemMediaItemReader interface {
GetByIDs(ctx context.Context, ids []string) ([]*models.MediaItem, error)
}
func ebookScanWorkers() int {
if v := os.Getenv("SILO_EBOOK_SCAN_WORKERS"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return audiobookScanWorkers()
}
func (s *Scanner) ScanEbookFolder(ctx context.Context, folder *models.MediaFolder) error {
if s == nil || folder == nil {
return fmt.Errorf("ScanEbookFolder: nil scanner or folder")
}
var candidates []string
for _, root := range folder.Paths {
if err := ctx.Err(); err != nil {
return err
}
walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
if err := ctx.Err(); err != nil {
return err
}
if walkErr != nil {
slog.Warn("ebook scan: walk error", "path", path, "error", walkErr)
return nil
}
if d.IsDir() {
return nil
}
if SupportsEbookFile(path) {
candidates = append(candidates, path)
}
return nil
})
if walkErr != nil {
slog.Warn("ebook scan: walk root failed", "root", root, "error", walkErr)
}
}
if len(candidates) == 0 {
return nil
}
workers := ebookScanWorkers()
slog.Info("ebook scan: starting",
"folder_id", folder.ID,
"candidates", len(candidates),
"workers", workers,
)
ch := make(chan string, workers*2)
var (
wg sync.WaitGroup
processed int64
failed int64
skipped int64
failMu sync.Mutex
failures []error
cancelErr error
)
start := time.Now()
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for path := range ch {
if ctx.Err() != nil {
return
}
if err := s.reconcileEbookFile(ctx, folder, path, &skipped); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
failMu.Lock()
if cancelErr == nil {
cancelErr = err
}
failMu.Unlock()
return
}
atomic.AddInt64(&failed, 1)
failMu.Lock()
failures = append(failures, fmt.Errorf("%s: %w", path, err))
failMu.Unlock()
slog.Warn("ebook scan: file failed",
"folder_id", folder.ID,
"path", path,
"error", err,
)
}
n := atomic.AddInt64(&processed, 1)
if n%500 == 0 {
slog.Info("ebook scan: progress",
"folder_id", folder.ID,
"processed", n,
"failed", atomic.LoadInt64(&failed),
"skipped", atomic.LoadInt64(&skipped),
"total", len(candidates),
"elapsed_sec", int(time.Since(start).Seconds()),
)
}
}
}()
}
for _, p := range candidates {
select {
case ch <- p:
case <-ctx.Done():
close(ch)
wg.Wait()
return ctx.Err()
}
}
close(ch)
wg.Wait()
if err := ctx.Err(); err != nil {
return err
}
if cancelErr != nil {
return cancelErr
}
slog.Info("ebook scan: completed",
"folder_id", folder.ID,
"processed", atomic.LoadInt64(&processed),
"failed", atomic.LoadInt64(&failed),
"skipped", atomic.LoadInt64(&skipped),
"elapsed_sec", int(time.Since(start).Seconds()),
)
if processedCount := atomic.LoadInt64(&processed); processedCount > 0 {
failedCount := atomic.LoadInt64(&failed)
skippedCount := atomic.LoadInt64(&skipped)
if failedCount > 0 && skippedCount == 0 && failedCount == processedCount {
return fmt.Errorf("ebook scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
}
}
return nil
}
func (s *Scanner) reconcileEbookFile(ctx context.Context, folder *models.MediaFolder, filePath string, skipped *int64) error {
info, err := os.Stat(filePath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("stat ebook file %s: %w", filePath, err)
}
size := info.Size()
modifiedAt := normalizeFileModifiedAt(info.ModTime())
isUnchanged, skipErr := s.ebookFileShouldSkip(ctx, folder, filePath, size, modifiedAt)
if skipErr != nil {
slog.Warn("ebook scan: skip-check failed, falling through",
"folder_id", folder.ID,
"path", filePath,
"error", skipErr,
)
} else if isUnchanged {
atomic.AddInt64(skipped, 1)
return nil
}
parsed, err := parseEbookFile(filePath)
if err != nil {
return fmt.Errorf("parse ebook file %s: %w", filePath, err)
}
if parsed.Title == "" {
parsed.Title = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
}
contentID, err := s.upsertEbookMediaItem(ctx, folder.ID, filePath, &parsed)
if err != nil {
return fmt.Errorf("upsert ebook item: %w", err)
}
if err := s.upsertEbookMediaFile(ctx, folder, contentID, filePath, size, modifiedAt, &parsed); err != nil {
return fmt.Errorf("upsert ebook file: %w", err)
}
if err := s.upsertEbookPeople(ctx, contentID, &parsed); err != nil {
return fmt.Errorf("upsert ebook people: %w", err)
}
if err := s.upsertEbookSeries(ctx, contentID, &parsed); err != nil {
return fmt.Errorf("upsert ebook series: %w", err)
}
if err := insertEbookLibraryMembership(ctx, s.fileRepo.Pool(), contentID, folder.ID); err != nil {
return fmt.Errorf("upsert ebook library membership: %w", err)
}
if parsed.ISBN != "" {
if err := insertEbookISBNProviderID(ctx, s.fileRepo.Pool(), contentID, parsed.ISBN); err != nil {
return fmt.Errorf("upsert ebook ISBN provider id: %w", err)
}
}
slog.Info("ebook scan: indexed",
"folder_id", folder.ID,
"content_id", contentID,
"title", parsed.Title,
"authors", parsed.Authors,
"path", filePath,
)
return nil
}
func (s *Scanner) ebookFileShouldSkip(ctx context.Context, folder *models.MediaFolder, filePath string, size int64, modifiedAt time.Time) (bool, error) {
if s.fileRepo == nil || s.itemRepo == nil {
return false, nil
}
existing, err := s.fileRepo.ListByObservedRootPath(ctx, folder.ID, filePath)
if err != nil {
return false, fmt.Errorf("list existing files: %w", err)
}
if len(existing) != 1 {
return false, nil
}
mf := existing[0]
if mf.FilePath != filePath || mf.FileSize != size || mf.FileModifiedAt == nil || !sameFileModifiedAt(mf.FileModifiedAt, modifiedAt) {
return false, nil
}
if mf.ContentID == "" {
return false, nil
}
statuses, err := s.itemRepo.GetStatusByIDs(ctx, []string{mf.ContentID})
if err != nil {
return false, fmt.Errorf("get item status: %w", err)
}
return !strings.EqualFold(strings.TrimSpace(statuses[mf.ContentID]), "unmatched"), nil
}
func (s *Scanner) upsertEbookMediaItem(ctx context.Context, folderID int, filePath string, book *parsedEbook) (string, error) {
if s.itemRepo == nil {
return "", fmt.Errorf("itemRepo not configured on Scanner")
}
if s.fileRepo == nil {
return "", fmt.Errorf("fileRepo not configured on Scanner")
}
existingID, err := s.fileRepo.FindContentIDByRootPath(ctx, folderID, filePath, "ebook")
if err != nil {
return "", fmt.Errorf("find ebook by root path: %w", err)
}
if existingID != "" {
if err := updateExistingEbookMediaItem(ctx, s.itemRepo, s.itemRepo, existingID, book); err != nil {
return "", err
}
return existingID, nil
}
if existing := s.findEbookByFilePath(ctx, filePath); existing != nil {
applyEbookToMediaItem(existing, book)
if existing.SortTitle == "" {
existing.SortTitle = titleutil.DeriveDefaultSortTitle(existing.Title)
}
if err := s.itemRepo.Upsert(ctx, existing); err != nil {
return "", err
}
return existing.ContentID, nil
}
return resolveEbookMediaItem(ctx, s.fileRepo, s.itemRepo, folderID, filePath, book)
}
func resolveEbookMediaItem(
ctx context.Context,
rootFinder filesystemRootContentFinder,
itemWriter filesystemMediaItemWriter,
folderID int,
filePath string,
book *parsedEbook,
) (string, error) {
if rootFinder == nil {
return "", fmt.Errorf("root content finder not configured")
}
if itemWriter == nil {
return "", fmt.Errorf("media item writer not configured")
}
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, filePath, "ebook")
if err != nil {
return "", fmt.Errorf("find ebook by root path: %w", err)
}
if existingID != "" {
return existingID, nil
}
cleanTitle := strings.TrimSpace(book.Title)
if cleanTitle == "" {
cleanTitle = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
}
return createEbookMediaItem(ctx, itemWriter, book, cleanTitle)
}
func createEbookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedEbook, cleanTitle string) (string, error) {
id, err := idgen.NextID()
if err != nil {
return "", fmt.Errorf("generate content_id: %w", err)
}
item := &models.MediaItem{
ContentID: id,
Type: "ebook",
Title: cleanTitle,
SortTitle: titleutil.DeriveDefaultSortTitle(cleanTitle),
Year: book.Year,
}
applyEbookToMediaItem(item, book)
if item.SortTitle == "" {
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
}
if err := itemWriter.Upsert(ctx, item); err != nil {
return "", err
}
return id, nil
}
func updateExistingEbookMediaItem(ctx context.Context, itemReader filesystemMediaItemReader, itemWriter filesystemMediaItemWriter, contentID string, book *parsedEbook) error {
if itemReader == nil {
return fmt.Errorf("media item reader not configured")
}
if itemWriter == nil {
return fmt.Errorf("media item writer not configured")
}
items, err := itemReader.GetByIDs(ctx, []string{contentID})
if err != nil {
return fmt.Errorf("get ebook media item %s: %w", contentID, err)
}
if len(items) == 0 || items[0] == nil {
return fmt.Errorf("ebook media item %s not found", contentID)
}
item := items[0]
applyEbookToMediaItem(item, book)
if item.SortTitle == "" {
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
}
return itemWriter.Upsert(ctx, item)
}
func applyEbookToMediaItem(item *models.MediaItem, book *parsedEbook) {
item.Type = "ebook"
if title := strings.TrimSpace(book.Title); title != "" {
item.Title = title
}
if book.Year > 0 {
item.Year = book.Year
}
if book.Description != "" && item.Overview == "" {
item.Overview = book.Description
}
if book.Publisher != "" {
item.Studios = mergeUniqueStrings(item.Studios, []string{book.Publisher})
}
if len(book.Genres) > 0 {
item.Genres = mergeUniqueStrings(item.Genres, book.Genres)
}
if !book.PublishedAt.IsZero() && item.ReleaseDate == nil {
rd := book.PublishedAt.UTC().Format("2006-01-02")
item.ReleaseDate = &rd
}
if book.Language != "" && item.OriginalLanguage == "" {
item.OriginalLanguage = book.Language
}
}
func (s *Scanner) findEbookByFilePath(ctx context.Context, filePath string) *models.MediaItem {
if s.fileRepo == nil {
return nil
}
var existingID string
err := s.fileRepo.Pool().QueryRow(ctx, `
SELECT mf.content_id
FROM media_files mf
JOIN media_items mi ON mi.content_id = mf.content_id
WHERE mf.file_path = $1
AND mi.type = 'ebook'
LIMIT 1
`, filePath).Scan(&existingID)
if err != nil || existingID == "" {
return nil
}
items, err := s.itemRepo.GetByIDs(ctx, []string{existingID})
if err != nil || len(items) == 0 {
return nil
}
return items[0]
}
func (s *Scanner) upsertEbookMediaFile(ctx context.Context, folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) error {
mf := buildEbookMediaFile(folder, contentID, filePath, size, modifiedAt, book)
if _, err := s.fileRepo.Upsert(ctx, mf); err != nil {
return fmt.Errorf("upsert media file %s: %w", filePath, err)
}
return nil
}
func buildEbookMediaFile(folder *models.MediaFolder, contentID string, filePath string, size int64, modifiedAt time.Time, book *parsedEbook) models.MediaFile {
return models.MediaFile{
ContentID: contentID,
MediaFolderID: folder.ID,
CanonicalRootPath: filePath,
ObservedRootPath: filePath,
ContentGroupKey: contentID,
GroupKeyVersion: 1,
BaseTitle: book.Title,
BaseYear: book.Year,
BaseType: "ebook",
IdentityConfidence: ebookIdentityConfidence(book),
FilePath: filePath,
FileSize: size,
FileModifiedAt: &modifiedAt,
Container: book.Format,
ProbeSource: "local",
}
}
func insertEbookLibraryMembership(ctx context.Context, exec ebookSQLExecutor, contentID string, folderID int) error {
_, err := exec.Exec(ctx, `
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
VALUES ($1, $2, NOW())
ON CONFLICT (content_id, media_folder_id) DO NOTHING
`, contentID, folderID)
return err
}
func insertEbookISBNProviderID(ctx context.Context, exec ebookSQLExecutor, contentID string, isbn string) error {
_, err := exec.Exec(ctx, `
INSERT INTO media_item_provider_ids (content_id, provider, provider_id, item_type)
VALUES ($1, 'isbn', $2, 'ebook')
ON CONFLICT DO NOTHING
`, contentID, isbn)
return err
}
type ebookCredit struct {
Name string
Kind models.PersonKind
}
func ebookPeopleCreditsEqual(existing []models.ItemPerson, desired []ebookCredit) bool {
existingAuthors := make([]models.ItemPerson, 0, len(existing))
for _, p := range existing {
if p.Kind == models.PersonKindAuthor {
existingAuthors = append(existingAuthors, p)
}
}
if len(existingAuthors) != len(desired) {
return false
}
type key struct {
name string
kind models.PersonKind
}
have := make(map[key]struct{}, len(existingAuthors))
for _, p := range existingAuthors {
have[key{strings.ToLower(strings.TrimSpace(p.Person.Name)), p.Kind}] = struct{}{}
}
for _, d := range desired {
k := key{strings.ToLower(strings.TrimSpace(d.Name)), d.Kind}
if _, ok := have[k]; !ok {
return false
}
}
return true
}
type ebookResolvedAuthor struct {
ID int64
Name string
}
func mergeEbookPeople(existing []models.ItemPerson, authors []ebookResolvedAuthor) []models.ItemPerson {
people := make([]models.ItemPerson, 0, len(existing)+len(authors))
for _, p := range existing {
if p.Kind == models.PersonKindAuthor {
continue
}
p.SortOrder = len(people)
people = append(people, p)
}
for _, author := range authors {
people = append(people, models.ItemPerson{
Person: models.Person{ID: author.ID, Name: author.Name},
Kind: models.PersonKindAuthor,
SortOrder: len(people),
})
}
return people
}
func ebookPeopleForReplace(existing []models.ItemPerson, getErr error, authors []ebookResolvedAuthor) ([]models.ItemPerson, error) {
if getErr != nil {
return nil, fmt.Errorf("get ebook people: %w", getErr)
}
return mergeEbookPeople(existing, authors), nil
}
func (s *Scanner) upsertEbookPeople(ctx context.Context, contentID string, book *parsedEbook) error {
if s.personRepo == nil {
return fmt.Errorf("personRepo not configured on Scanner")
}
if s.itemRepo == nil {
return fmt.Errorf("itemRepo not configured on Scanner")
}
var desired []ebookCredit
for _, author := range book.Authors {
if name := strings.TrimSpace(author); name != "" {
desired = append(desired, ebookCredit{Name: name, Kind: models.PersonKindAuthor})
}
}
if len(desired) == 0 {
return nil
}
existing, err := s.itemRepo.GetPeople(ctx, contentID)
if err != nil {
return fmt.Errorf("get ebook people: %w", err)
}
if ebookPeopleCreditsEqual(existing, desired) {
return nil
}
authors := make([]ebookResolvedAuthor, 0, len(desired))
for _, c := range desired {
personID, err := s.personRepo.FindOrCreate(ctx, models.Person{Name: c.Name})
if err != nil {
return fmt.Errorf("find-or-create person %q: %w", c.Name, err)
}
authors = append(authors, ebookResolvedAuthor{ID: personID, Name: c.Name})
}
people, err := ebookPeopleForReplace(existing, nil, authors)
if err != nil {
return err
}
return s.itemRepo.ReplacePeople(ctx, contentID, people)
}
func (s *Scanner) upsertEbookSeries(ctx context.Context, contentID string, book *parsedEbook) error {
if s == nil {
return fmt.Errorf("Scanner not configured")
}
if s.fileRepo == nil {
return fmt.Errorf("fileRepo not configured on Scanner")
}
var currentName *string
var currentIdx *float64
err := s.fileRepo.Pool().QueryRow(ctx, `
SELECT series_name, series_index FROM ebook_series WHERE content_id = $1
`, contentID).Scan(&currentName, &currentIdx)
plan, err := planEbookSeriesWrite(book, currentName, currentIdx, err)
if err != nil {
return err
}
switch plan.Kind {
case ebookSeriesWriteNone:
return nil
case ebookSeriesWriteDelete:
if _, delErr := s.fileRepo.Pool().Exec(ctx,
`DELETE FROM ebook_series WHERE content_id = $1`, contentID); delErr != nil {
return fmt.Errorf("delete ebook_series row: %w", delErr)
}
return nil
case ebookSeriesWriteUpsert:
var idx any
if plan.Index != nil {
idx = *plan.Index
}
if _, err := s.fileRepo.Pool().Exec(ctx, `
INSERT INTO ebook_series (content_id, series_name, series_index, updated_at)
VALUES ($1, $2, $3, NOW())
ON CONFLICT (content_id) DO UPDATE SET
series_name = EXCLUDED.series_name,
series_index = EXCLUDED.series_index,
updated_at = NOW()
`, contentID, plan.Name, idx); err != nil {
return fmt.Errorf("upsert ebook_series row: %w", err)
}
return nil
default:
return fmt.Errorf("unknown ebook_series write kind: %d", plan.Kind)
}
}
func ebookSeriesDesired(book *parsedEbook) (string, *float64) {
if book == nil {
return "", nil
}
return strings.TrimSpace(book.Series), parseSeriesIndex(book.SeriesIndex)
}
type ebookSeriesWriteKind int
const (
ebookSeriesWriteNone ebookSeriesWriteKind = iota
ebookSeriesWriteDelete
ebookSeriesWriteUpsert
)
type ebookSeriesWritePlan struct {
Kind ebookSeriesWriteKind
Name string
Index *float64
}
func planEbookSeriesWrite(book *parsedEbook, currentName *string, currentIdx *float64, queryErr error) (ebookSeriesWritePlan, error) {
if queryErr != nil {
if !errors.Is(queryErr, pgx.ErrNoRows) {
return ebookSeriesWritePlan{}, fmt.Errorf("query ebook_series: %w", queryErr)
}
currentName = nil
currentIdx = nil
}
desiredName, desiredIdx := ebookSeriesDesired(book)
if desiredName == "" {
if currentName == nil {
return ebookSeriesWritePlan{Kind: ebookSeriesWriteNone}, nil
}
return ebookSeriesWritePlan{Kind: ebookSeriesWriteDelete}, nil
}
if currentName != nil && *currentName == desiredName && floatPtrEqual(currentIdx, desiredIdx) {
return ebookSeriesWritePlan{Kind: ebookSeriesWriteNone}, nil
}
return ebookSeriesWritePlan{
Kind: ebookSeriesWriteUpsert,
Name: desiredName,
Index: desiredIdx,
}, nil
}
func ebookIdentityConfidence(book *parsedEbook) string {
if book == nil {
return "low"
}
score := 0
if book.Title != "" {
score++
}
if len(book.Authors) > 0 {
score++
}
if book.Year > 0 {
score++
}
if book.ISBN != "" {
score++
}
switch {
case score >= 4:
return "high"
case score > 0:
return "medium"
default:
return "low"
}
}
+590
View File
@@ -0,0 +1,590 @@
package scanner
import (
"archive/zip"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type recordingEbookExecutor struct {
queries []string
args [][]any
}
func (r *recordingEbookExecutor) Exec(_ context.Context, query string, args ...any) (pgconn.CommandTag, error) {
r.queries = append(r.queries, query)
r.args = append(r.args, append([]any(nil), args...))
return pgconn.CommandTag{}, nil
}
type fakeFilesystemItemReader struct {
items []*models.MediaItem
err error
}
func (f *fakeFilesystemItemReader) GetByIDs(_ context.Context, _ []string) ([]*models.MediaItem, error) {
return f.items, f.err
}
func TestSupportsEbookFile(t *testing.T) {
cases := []struct {
path string
want bool
}{
{"book.epub", true},
{"book.PDF", true},
{"book.mobi", true},
{"book.azw", true},
{"book.azw3", true},
{"book.FB2", true},
{"book.mp3", false},
{"movie.mkv", false},
}
for _, tc := range cases {
if got := SupportsEbookFile(tc.path); got != tc.want {
t.Errorf("SupportsEbookFile(%q) = %v, want %v", tc.path, got, tc.want)
}
}
}
func TestNormalizeEbookISBNHandlesCommonLabels(t *testing.T) {
cases := []struct {
in string
want string
}{
{"ISBN-13: 978-0-306-40615-7", "9780306406157"},
{"ISBN-10: 0-9752298-0-x", "097522980X"},
}
for _, tc := range cases {
if got := normalizeEbookISBN(tc.in); got != tc.want {
t.Errorf("normalizeEbookISBN(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
func TestParseEbookEPUBMetadata(t *testing.T) {
path := writeTestEPUB(t, []string{"ISBN: 978-0-306-40615-7"})
got, err := parseEbookFile(path)
if err != nil {
t.Fatalf("parseEbookFile: %v", err)
}
if got.Format != "epub" {
t.Fatalf("Format = %q, want epub", got.Format)
}
if got.Title != "The Test Ebook" {
t.Fatalf("Title = %q, want The Test Ebook", got.Title)
}
if strings.Join(got.Authors, ", ") != "Ada Writer, Ben Author" {
t.Fatalf("Authors = %v, want Ada Writer and Ben Author", got.Authors)
}
if got.ISBN != "9780306406157" {
t.Fatalf("ISBN = %q, want normalized ISBN", got.ISBN)
}
if got.Year != 2024 {
t.Fatalf("Year = %d, want 2024", got.Year)
}
if got.Publisher != "Silo Press" {
t.Fatalf("Publisher = %q, want Silo Press", got.Publisher)
}
wantPublishedAt := time.Date(2024, 3, 10, 0, 0, 0, 0, time.UTC)
if !got.PublishedAt.Equal(wantPublishedAt) {
t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, wantPublishedAt)
}
if got.Language != "en" {
t.Fatalf("Language = %q, want en", got.Language)
}
if strings.Join(got.Genres, ", ") != "Fiction, Adventure" {
t.Fatalf("Genres = %v, want Fiction and Adventure", got.Genres)
}
if got.Series != "Test Series" || got.SeriesIndex != "2" {
t.Fatalf("Series = %q/%q, want Test Series/2", got.Series, got.SeriesIndex)
}
}
func TestParseEbookEPUBSkipsUUIDIdentifierBeforeISBN(t *testing.T) {
path := writeTestEPUB(t, []string{
"550e8400-e29b-41d4-a716-446655440000",
"ISBN: 978-0-306-40615-7",
})
got, err := parseEbookFile(path)
if err != nil {
t.Fatalf("parseEbookFile: %v", err)
}
if got.ISBN != "9780306406157" {
t.Fatalf("ISBN = %q, want normalized ISBN after skipping UUID", got.ISBN)
}
}
func TestReadEPUBZipEntryRejectsOversizedEntry(t *testing.T) {
path := filepath.Join(t.TempDir(), "large.epub")
file, err := os.Create(path)
if err != nil {
t.Fatalf("create epub: %v", err)
}
zw := zip.NewWriter(file)
w, err := zw.Create("OPS/content.opf")
if err != nil {
t.Fatalf("create zip entry: %v", err)
}
if _, err := w.Write([]byte(strings.Repeat("x", maxEPUBMetadataEntrySize+1))); err != nil {
t.Fatalf("write zip entry: %v", err)
}
if err := zw.Close(); err != nil {
t.Fatalf("close zip: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("close file: %v", err)
}
reader, err := zip.OpenReader(path)
if err != nil {
t.Fatalf("open zip: %v", err)
}
defer reader.Close()
_, err = readEPUBZipEntry(&reader.Reader, "OPS/content.opf")
if err == nil || !strings.Contains(err.Error(), "epub entry too large") {
t.Fatalf("readEPUBZipEntry error = %v, want size limit error", err)
}
}
func TestParseEbookFB2Metadata(t *testing.T) {
path := filepath.Join(t.TempDir(), "book.fb2")
if err := os.WriteFile(path, []byte(`<?xml version="1.0" encoding="UTF-8"?>
<FictionBook>
<description>
<title-info>
<genre>science fiction</genre>
<author><first-name>Ada</first-name><last-name>Writer</last-name></author>
<book-title>FB2 Test Ebook</book-title>
<date value="2022-04-05">5 April 2022</date>
<lang>en</lang>
<sequence name="FB2 Series" number="3"/>
</title-info>
<publish-info>
<publisher>FB2 Press</publisher>
<year>2022</year>
<isbn>978-0-306-40615-7</isbn>
</publish-info>
</description>
</FictionBook>`), 0o644); err != nil {
t.Fatalf("write fb2: %v", err)
}
got, err := parseEbookFile(path)
if err != nil {
t.Fatalf("parseEbookFile: %v", err)
}
if got.Format != "fb2" || got.Title != "FB2 Test Ebook" {
t.Fatalf("Format/Title = %q/%q, want fb2/FB2 Test Ebook", got.Format, got.Title)
}
if strings.Join(got.Authors, ", ") != "Ada Writer" {
t.Fatalf("Authors = %v, want Ada Writer", got.Authors)
}
if got.ISBN != "9780306406157" {
t.Fatalf("ISBN = %q, want normalized ISBN", got.ISBN)
}
if got.Publisher != "FB2 Press" {
t.Fatalf("Publisher = %q, want FB2 Press", got.Publisher)
}
if got.Year != 2022 {
t.Fatalf("Year = %d, want 2022", got.Year)
}
wantPublishedAt := time.Date(2022, 4, 5, 0, 0, 0, 0, time.UTC)
if !got.PublishedAt.Equal(wantPublishedAt) {
t.Fatalf("PublishedAt = %v, want %v", got.PublishedAt, wantPublishedAt)
}
if got.Language != "en" {
t.Fatalf("Language = %q, want en", got.Language)
}
if strings.Join(got.Genres, ", ") != "science fiction" {
t.Fatalf("Genres = %v, want science fiction", got.Genres)
}
if got.Series != "FB2 Series" || got.SeriesIndex != "3" {
t.Fatalf("Series = %q/%q, want FB2 Series/3", got.Series, got.SeriesIndex)
}
}
func TestEbookSeriesDesiredParsesIndex(t *testing.T) {
name, idx := ebookSeriesDesired(&parsedEbook{
Series: " The Expanse ",
SeriesIndex: "2 of 9",
})
if name != "The Expanse" {
t.Fatalf("series name = %q, want The Expanse", name)
}
if idx == nil || *idx != 2 {
t.Fatalf("series index = %v, want 2", idx)
}
}
func TestUpsertEbookSeriesNilScannerReturnsError(t *testing.T) {
var s *Scanner
if err := s.upsertEbookSeries(context.Background(), "content-1", &parsedEbook{Series: "Series"}); err == nil {
t.Fatal("upsertEbookSeries nil scanner error = nil, want error")
}
}
func TestUpsertEbookSeriesNilFileRepoReturnsError(t *testing.T) {
s := &Scanner{}
if err := s.upsertEbookSeries(context.Background(), "content-1", &parsedEbook{Series: "Series"}); err == nil {
t.Fatal("upsertEbookSeries nil fileRepo error = nil, want error")
}
}
func TestPlanEbookSeriesWriteInsertsWhenRowAbsent(t *testing.T) {
plan, err := planEbookSeriesWrite(&parsedEbook{Series: " Series ", SeriesIndex: "2 of 9"}, nil, nil, pgx.ErrNoRows)
if err != nil {
t.Fatalf("planEbookSeriesWrite: %v", err)
}
if plan.Kind != ebookSeriesWriteUpsert || plan.Name != "Series" {
t.Fatalf("plan = %+v, want upsert Series", plan)
}
if plan.Index == nil || *plan.Index != 2 {
t.Fatalf("index = %v, want 2", plan.Index)
}
}
func TestPlanEbookSeriesWriteBlankSeriesDeletesExistingAndSkipsAbsent(t *testing.T) {
currentName := "Series"
plan, err := planEbookSeriesWrite(&parsedEbook{Series: " "}, &currentName, nil, nil)
if err != nil {
t.Fatalf("planEbookSeriesWrite delete: %v", err)
}
if plan.Kind != ebookSeriesWriteDelete {
t.Fatalf("blank existing plan = %+v, want delete", plan)
}
plan, err = planEbookSeriesWrite(&parsedEbook{Series: ""}, nil, nil, pgx.ErrNoRows)
if err != nil {
t.Fatalf("planEbookSeriesWrite absent: %v", err)
}
if plan.Kind != ebookSeriesWriteNone {
t.Fatalf("blank absent plan = %+v, want none", plan)
}
}
func TestPlanEbookSeriesWriteSkipsIdenticalRow(t *testing.T) {
currentName := "Series"
currentIdx := 3.5
plan, err := planEbookSeriesWrite(&parsedEbook{Series: "Series", SeriesIndex: "3.5"}, &currentName, &currentIdx, nil)
if err != nil {
t.Fatalf("planEbookSeriesWrite: %v", err)
}
if plan.Kind != ebookSeriesWriteNone {
t.Fatalf("identical plan = %+v, want none", plan)
}
}
func TestPlanEbookSeriesWriteAllowsNullNumericIndex(t *testing.T) {
plan, err := planEbookSeriesWrite(&parsedEbook{Series: "Series", SeriesIndex: "appendix"}, nil, nil, pgx.ErrNoRows)
if err != nil {
t.Fatalf("planEbookSeriesWrite: %v", err)
}
if plan.Kind != ebookSeriesWriteUpsert || plan.Index != nil {
t.Fatalf("plan = %+v, want upsert with nil index", plan)
}
}
func TestPlanEbookSeriesWriteReturnsQueryError(t *testing.T) {
queryErr := errors.New("query failed")
if _, err := planEbookSeriesWrite(&parsedEbook{Series: "Series"}, nil, nil, queryErr); !errors.Is(err, queryErr) {
t.Fatalf("planEbookSeriesWrite error = %v, want query error", err)
}
}
func TestEbookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
book := &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
if got := ebookIdentityConfidence(book); got != "high" {
t.Fatalf("complete metadata confidence = %q, want high", got)
}
book = &parsedEbook{Title: "Tagged Ebook", Authors: []string{"Author"}, Year: 2024}
if got := ebookIdentityConfidence(book); got != "medium" {
t.Fatalf("partial metadata confidence = %q, want medium", got)
}
book = &parsedEbook{}
if got := ebookIdentityConfidence(book); got != "low" {
t.Fatalf("empty metadata confidence = %q, want low", got)
}
}
func TestResolveEbookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
finder := &fakeRootContentFinder{}
writer := &fakeFilesystemItemWriter{}
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{
Title: "Book", Year: 2024, Authors: []string{"Author"}, Description: "Overview", Publisher: "Publisher", Genres: []string{"Fiction"}, Language: "en",
})
if err != nil {
t.Fatalf("resolveEbookMediaItem: %v", err)
}
if got == "" || len(writer.upserts) != 1 {
t.Fatalf("contentID/upserts = %q/%d, want id and one upsert", got, len(writer.upserts))
}
item := writer.upserts[0]
if item.Type != "ebook" || item.Title != "Book" || item.Year != 2024 || item.Overview != "Overview" || item.OriginalLanguage != "en" {
t.Fatalf("upserted ebook item = %+v", item)
}
}
func TestResolveEbookMediaItemReusesRootScopedContentID(t *testing.T) {
finder := &fakeRootContentFinder{contentID: "ebook-root-id"}
writer := &fakeFilesystemItemWriter{}
got, err := resolveEbookMediaItem(context.Background(), finder, writer, 7, "/library/Author/Book.epub", &parsedEbook{Title: "Book"})
if err != nil {
t.Fatalf("resolveEbookMediaItem: %v", err)
}
if got != "ebook-root-id" {
t.Fatalf("contentID = %q, want root-scoped id", got)
}
if len(writer.upserts) != 0 {
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
}
}
func TestResolveEbookExistingRootAppliesParsedMetadata(t *testing.T) {
reader := &fakeFilesystemItemReader{items: []*models.MediaItem{{
ContentID: "ebook-root-id",
Type: "ebook",
Title: "Old Title",
Year: 2020,
OriginalLanguage: "",
}}}
writer := &fakeFilesystemItemWriter{}
err := updateExistingEbookMediaItem(context.Background(), reader, writer, "ebook-root-id", &parsedEbook{
Title: "New Title",
Year: 2026,
Publisher: "New Press",
Genres: []string{"Fiction"},
Language: "en",
})
if err != nil {
t.Fatalf("updateExistingEbookMediaItem: %v", err)
}
if len(writer.upserts) != 1 {
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
}
item := writer.upserts[0]
if item.ContentID != "ebook-root-id" || item.Type != "ebook" || item.Title != "New Title" || item.Year != 2026 || item.OriginalLanguage != "en" {
t.Fatalf("updated item = %+v", item)
}
if strings.Join(item.Studios, ",") != "New Press" || strings.Join(item.Genres, ",") != "Fiction" {
t.Fatalf("updated item studios/genres = %+v/%+v", item.Studios, item.Genres)
}
}
func TestApplyEbookToMediaItemPreservesExistingYearWhenParsedYearMissing(t *testing.T) {
item := &models.MediaItem{
Type: "ebook",
Title: "Existing",
Year: 1999,
}
applyEbookToMediaItem(item, &parsedEbook{Title: "Updated", Year: 0})
if item.Title != "Updated" {
t.Fatalf("Title = %q, want Updated", item.Title)
}
if item.Year != 1999 {
t.Fatalf("Year = %d, want preserved 1999", item.Year)
}
}
func TestEbookPeopleCreditsEqualAuthorsOnly(t *testing.T) {
existing := []models.ItemPerson{
{Person: models.Person{Name: "Ada Writer"}, Kind: models.PersonKindAuthor, SortOrder: 0},
{Person: models.Person{Name: "Ben Author"}, Kind: models.PersonKindAuthor, SortOrder: 1},
{Person: models.Person{Name: "Manual Contributor"}, Kind: models.PersonKindWriter, SortOrder: 2},
}
desired := []ebookCredit{
{Name: "Ada Writer", Kind: models.PersonKindAuthor},
{Name: "Ben Author", Kind: models.PersonKindAuthor},
}
if !ebookPeopleCreditsEqual(existing, desired) {
t.Fatal("expected matching author credits to compare equal while ignoring non-authors")
}
existing[1] = models.ItemPerson{Person: models.Person{Name: "Other Author"}, Kind: models.PersonKindAuthor, SortOrder: 1}
if ebookPeopleCreditsEqual(existing, desired) {
t.Fatal("expected different author credit to make ebook author set differ")
}
}
func TestEbookPeopleMergePreservesExistingNonAuthorCredits(t *testing.T) {
existing := []models.ItemPerson{
{Person: models.Person{ID: 10, Name: "Old Author"}, Kind: models.PersonKindAuthor, SortOrder: 0},
{Person: models.Person{ID: 20, Name: "Manual Writer"}, Kind: models.PersonKindWriter, SortOrder: 1, Character: "essay"},
{Person: models.Person{ID: 30, Name: "Manual Narrator"}, Kind: models.PersonKindNarrator, SortOrder: 2},
}
authors := []ebookResolvedAuthor{
{ID: 40, Name: "New Author"},
}
got := mergeEbookPeople(existing, authors)
if len(got) != 3 {
t.Fatalf("merged people len = %d, want 3: %+v", len(got), got)
}
if got[0].Person.ID != 20 || got[0].Kind != models.PersonKindWriter || got[0].Character != "essay" {
t.Fatalf("first preserved non-author = %+v", got[0])
}
if got[1].Person.ID != 30 || got[1].Kind != models.PersonKindNarrator {
t.Fatalf("second preserved non-author = %+v", got[1])
}
if got[2].Person.ID != 40 || got[2].Kind != models.PersonKindAuthor || got[2].SortOrder != 2 {
t.Fatalf("new author credit = %+v", got[2])
}
}
func TestEbookPeopleReplacePlanReturnsGetPeopleError(t *testing.T) {
wantErr := errors.New("get people failed")
_, err := ebookPeopleForReplace(nil, wantErr, []ebookResolvedAuthor{{ID: 40, Name: "New Author"}})
if !errors.Is(err, wantErr) {
t.Fatalf("error = %v, want %v", err, wantErr)
}
}
func TestScanEbookBuildMediaFileSetsCorePersistenceFields(t *testing.T) {
modifiedAt := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
book := &parsedEbook{Format: "epub", Title: "Book", Authors: []string{"Author"}, Year: 2024, ISBN: "9780306406157"}
got := buildEbookMediaFile(&models.MediaFolder{ID: 44}, "content-1", "/library/Book.epub", 1234, modifiedAt, book)
if got.ContentID != "content-1" || got.MediaFolderID != 44 {
t.Fatalf("ids = %q/%d, want content-1/44", got.ContentID, got.MediaFolderID)
}
if got.BaseType != "ebook" || got.BaseTitle != "Book" || got.BaseYear != 2024 || got.Container != "epub" || got.ProbeSource != "local" {
t.Fatalf("ebook file metadata = %+v", got)
}
if got.CanonicalRootPath != "/library/Book.epub" || got.ObservedRootPath != "/library/Book.epub" || got.FilePath != "/library/Book.epub" {
t.Fatalf("ebook paths = canonical %q observed %q file %q", got.CanonicalRootPath, got.ObservedRootPath, got.FilePath)
}
if got.FileSize != 1234 || got.FileModifiedAt == nil || !got.FileModifiedAt.Equal(modifiedAt) {
t.Fatalf("file facts = size %d modified %v", got.FileSize, got.FileModifiedAt)
}
}
func TestScanEbookPersistenceSQLWritesLibraryMembershipAndISBNOnly(t *testing.T) {
ctx := context.Background()
exec := &recordingEbookExecutor{}
if err := insertEbookLibraryMembership(ctx, exec, "content-1", 44); err != nil {
t.Fatalf("insertEbookLibraryMembership: %v", err)
}
if err := insertEbookISBNProviderID(ctx, exec, "content-1", "9780306406157"); err != nil {
t.Fatalf("insertEbookISBNProviderID: %v", err)
}
if len(exec.queries) != 2 {
t.Fatalf("queries = %d, want 2", len(exec.queries))
}
if !strings.Contains(exec.queries[0], "media_item_libraries") || exec.args[0][0] != "content-1" || exec.args[0][1] != 44 {
t.Fatalf("library membership write = query %q args %+v", exec.queries[0], exec.args[0])
}
if !strings.Contains(exec.queries[1], "media_item_provider_ids") || !strings.Contains(exec.queries[1], "'isbn'") || !strings.Contains(exec.queries[1], "'ebook'") {
t.Fatalf("ISBN provider write query = %q", exec.queries[1])
}
if strings.Contains(exec.queries[1], "asin") || exec.args[1][0] != "content-1" || exec.args[1][1] != "9780306406157" {
t.Fatalf("ISBN provider write args/query = query %q args %+v", exec.queries[1], exec.args[1])
}
}
func TestScanEbookFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "bad.epub"), []byte("not a real epub"), 0o644); err != nil {
t.Fatalf("write fake ebook: %v", err)
}
s := &Scanner{}
err := s.ScanEbookFolder(context.Background(), &models.MediaFolder{ID: 44, Paths: []string{root}})
if err == nil {
t.Fatal("ScanEbookFolder returned nil, want aggregate failure")
}
if !strings.Contains(err.Error(), "folder_id=44") {
t.Fatalf("error = %q, want folder id", err)
}
}
func TestScanEbookFolderReturnsCanceledContext(t *testing.T) {
root := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := &Scanner{}
err := s.ScanEbookFolder(ctx, &models.MediaFolder{ID: 44, Paths: []string{root}})
if !errors.Is(err, context.Canceled) {
t.Fatalf("ScanEbookFolder error = %v, want context.Canceled", err)
}
}
func writeTestEPUB(t *testing.T, identifiers []string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "book.epub")
file, err := os.Create(path)
if err != nil {
t.Fatalf("create epub: %v", err)
}
zw := zip.NewWriter(file)
add := func(name, body string) {
t.Helper()
w, err := zw.Create(name)
if err != nil {
t.Fatalf("create zip entry %s: %v", name, err)
}
if _, err := w.Write([]byte(body)); err != nil {
t.Fatalf("write zip entry %s: %v", name, err)
}
}
add("META-INF/container.xml", `<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
<rootfiles>
<rootfile full-path="OPS/content.opf" media-type="application/oebps-package+xml"/>
</rootfiles>
</container>`)
var identifierXML strings.Builder
for _, identifier := range identifiers {
identifierXML.WriteString(" <dc:identifier>")
identifierXML.WriteString(identifier)
identifierXML.WriteString("</dc:identifier>\n")
}
add("OPS/content.opf", `<?xml version="1.0" encoding="UTF-8"?>
<package xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
<metadata>
<dc:title>The Test Ebook</dc:title>
<dc:creator>Ada Writer</dc:creator>
<dc:creator>Ben Author</dc:creator>
`+identifierXML.String()+`
<dc:publisher>Silo Press</dc:publisher>
<dc:date>2024-03-10</dc:date>
<dc:language>en</dc:language>
<dc:subject>Fiction</dc:subject>
<dc:subject>Adventure</dc:subject>
<dc:description>Back cover copy</dc:description>
<meta name="calibre:series" content="Test Series"/>
<meta name="calibre:series_index" content="2"/>
</metadata>
</package>`)
if err := zw.Close(); err != nil {
t.Fatalf("close zip: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("close epub: %v", err)
}
return path
}
+26
View File
@@ -241,15 +241,27 @@ func (s *Scanner) ScanFolder(ctx context.Context, folder *models.MediaFolder) (*
return &ScanResult{}, nil
}
if isEbookLibraryType(folder.Type) {
if err := s.ScanEbookFolder(watchCtx, folder); err != nil {
return nil, err
}
return &ScanResult{}, nil
}
return s.scanPaths(watchCtx, folder, folder.Paths, folder.Paths, true)
}
var errEbookScanningNotImplemented = errors.New("ebook scanning is not implemented")
// ScanSubtree walks a single subtree within a media folder and reconciles only
// files that live beneath that subtree.
func (s *Scanner) ScanSubtree(ctx context.Context, folder *models.MediaFolder, subtreePath string) (*ScanResult, error) {
cleanSubtree := filepath.Clean(subtreePath)
watchCtx, stopWatch := s.watchFolderContext(ctx, folder.ID)
defer stopWatch()
if isEbookLibraryType(folder.Type) {
return nil, errEbookScanningNotImplemented
}
return s.scanPaths(watchCtx, folder, []string{cleanSubtree}, []string{cleanSubtree}, false)
}
@@ -278,6 +290,15 @@ func isPodcastLibraryType(libraryType string) bool {
}
}
func isEbookLibraryType(libraryType string) bool {
switch strings.ToLower(strings.TrimSpace(libraryType)) {
case "ebook", "ebooks":
return true
default:
return false
}
}
// walkMode tells walkLogicalTree which file extensions to surface and
// which library-specific filename heuristics (sample/extra skipping)
// to apply.
@@ -288,6 +309,7 @@ const (
walkModeMovie // movie library: video extensions + sample/extra skipping
walkModeAudiobook // audiobook library: audio extensions, no skipping
walkModePodcast // podcast library: audio extensions, no skipping
walkModeEbook // ebook library: ebook extensions, no skipping
)
// walkModeFor derives a walkMode from a media_folders.type string.
@@ -301,6 +323,8 @@ func walkModeFor(folderType string) walkMode {
return walkModeAudiobook
case isPodcastLibraryType(folderType):
return walkModePodcast
case isEbookLibraryType(folderType):
return walkModeEbook
default:
return walkModeVideo
}
@@ -312,6 +336,8 @@ func (m walkMode) acceptsExt(ext string) bool {
switch m {
case walkModeAudiobook, walkModePodcast:
return audioExtensions[ext]
case walkModeEbook:
return ebookExtensions[ext]
default:
return videoExtensions[ext]
}
+80
View File
@@ -2,6 +2,7 @@ package scanner
import (
"context"
"errors"
"os"
"path/filepath"
"sort"
@@ -200,3 +201,82 @@ func TestIsPodcastLibraryType(t *testing.T) {
}
}
}
func TestIsEbookLibraryType(t *testing.T) {
cases := []struct {
in string
want bool
}{
{"ebooks", true},
{"ebook", true},
{"Ebook", true},
{" EBOOKS ", true},
{"audiobooks", false},
{"movies", false},
{"", false},
}
for _, tc := range cases {
if got := isEbookLibraryType(tc.in); got != tc.want {
t.Errorf("isEbookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
}
}
}
func TestWalkModeForEbookLibraryTypes(t *testing.T) {
for _, libraryType := range []string{"ebook", "ebooks", " EBOOKS "} {
if got := walkModeFor(libraryType); got != walkModeEbook {
t.Fatalf("walkModeFor(%q) = %v, want %v", libraryType, got, walkModeEbook)
}
}
}
func TestWalkModeEbookAcceptsEbookExtensionsOnly(t *testing.T) {
if !walkModeEbook.acceptsExt(".epub") {
t.Fatal("walkModeEbook should accept .epub")
}
if !walkModeEbook.acceptsExt(".pdf") {
t.Fatal("walkModeEbook should accept .pdf")
}
if walkModeEbook.acceptsExt(".mp4") {
t.Fatal("walkModeEbook should reject video extensions")
}
if walkModeEbook.acceptsExt(".mp3") {
t.Fatal("walkModeEbook should reject audio extensions")
}
}
func TestScanFolderEbookLibraryRoutesToEbookScanner(t *testing.T) {
scanner := &Scanner{}
result, err := scanner.ScanFolder(context.Background(), &models.MediaFolder{
ID: 0,
Type: " ebooks ",
Paths: []string{t.TempDir()},
})
if err != nil {
t.Fatalf("ScanFolder ebook error = %v, want nil", err)
}
if result == nil {
t.Fatal("ScanFolder ebook result = nil, want empty result")
}
}
func TestScanSubtreeEbookLibraryReturnsNotImplemented(t *testing.T) {
subtree := t.TempDir()
ebookPath := filepath.Join(subtree, "Book.epub")
if err := os.WriteFile(ebookPath, []byte("not a real epub"), 0o644); err != nil {
t.Fatalf("write ebook fixture: %v", err)
}
scanner := &Scanner{}
result, err := scanner.ScanSubtree(context.Background(), &models.MediaFolder{
ID: 0,
Type: "ebook",
Paths: []string{subtree},
}, subtree)
if !errors.Is(err, errEbookScanningNotImplemented) {
t.Fatalf("ScanSubtree ebook error = %v, want %v", err, errEbookScanningNotImplemented)
}
if result != nil {
t.Fatalf("ScanSubtree ebook result = %+v, want nil", result)
}
}
@@ -0,0 +1,17 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE ebook_series (
content_id TEXT PRIMARY KEY REFERENCES media_items(content_id) ON DELETE CASCADE,
series_name TEXT NOT NULL,
series_index NUMERIC,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX ebook_series_name_lower
ON ebook_series (LOWER(series_name), series_index NULLS LAST);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS ebook_series;
-- +goose StatementEnd