Files
silo-server/internal/scanner/ebook.go
T
8b70357703 feat(ebooks): first-class ebook libraries, scanner, and reader (#124)
* docs: define ebook architecture matching audiobooks

* docs: plan ebook audiobook-parity implementation

* feat: add ebook scanner parser foundation

* fix: harden ebook scanner foundation

* fix: handle ebook isbn labels

* fix: guard ebook subtree scans

* feat: scan ebook libraries in core

* fix: preserve ebook scan people credits

* fix: refresh ebook scan metadata safely

* feat: persist ebook series membership

* test: cover ebook series persistence decisions

* fix: address ebook scanner PR review

* docs: clarify ebook foundation PR scope

* feat: add ebook metadata enricher

* fix: harden ebook poster cache

* feat: wire ebook metadata sync task

* feat: expose ebook library metadata setup

* feat: add ebook catalog scope support

* feat: add ebook detail view

* feat: label ebook file versions by format

* feat: use file-size copy for downloads

* feat: use file language in download dialog

* test: cover ebook detail authors and downloads

* fix: drop narrator credits from ebook scanner merges

* fix: align ebook collection filters with book media

* fix: drop asin provider ids from ebook enrichment

* fix: force ebook people refresh for stale narrators

* chore: omit ebook planning docs from branch

* feat: add ebook detail related content

* feat: add ebook reader file entrypoint

* feat: render ebooks with foliate reader

* feat: persist ebook reader progress

* feat: add ebook reader controls

* feat: extract ebook pdf metadata

* feat: favor scanner isbn during ebook enrichment

* feat: extract fbz ebook metadata

* feat: count cbz ebook pages

* feat: show ebook file page counts

* feat: show ebook download summaries

* feat: switch ebook reader files

* feat: prefer epub for ebook read action

* feat: surface ebook reader progress

* feat: sync ebook reader progress cache

* feat: hide ebook read action for unsupported files

* feat: filter ebook reader file selector

* fix: serve fbz ebook archives with reader mime type

* fix: detect fbz ebooks from compound filename

* fix: authorize fbz ebooks from compound filename

* fix: scope ebook catalog facets

* fix: reject narrator queries for ebooks

* fix: build ebook recommendation text from authors

* fix: include ebooks in embedding eligibility

* fix: include ebooks in recommendation media mix

* fix: include ebooks in recently added recommendations

* feat: include ebook progress in recommendation signals

* feat: include ebooks in continue watching sections

* feat: include ebooks in catalog progress metrics

* fix: read ebook isbn from epub metadata

* fix: filter ebook asin provider aliases

* fix: fall back from unsupported ebook reader files

* fix: sort ebook catalogs by reader progress

* fix: filter ebook catalogs by reader progress

* fix: include ebooks in last watched catalog filters

* feat: reflect ebook reader progress in item user state

* feat: share ebook progress state across item surfaces

* feat: report ebook scan progress

* fix: include ebook activity in recommendations

* fix: expose ebook reader progress on item detail

* fix: support ebook subtree scans

* fix: honor profile header for ebook item progress

* fix: add ebook library default sections

* fix: route ebook continue cards to reader

* fix: hide watched toggle for ebooks

* fix: route ebook watch tonight cards to reader

* fix: route ebook hero actions to reader

* fix: detect archive ebook reader formats by filename

* feat: cache embedded ebook covers during scan

* fix: encode ebook hero reader links

* fix: persist non-epub ebook reader progress

* fix: scope narrator catalog badges to audiobooks

* fix: merge ebook reader progress during item repair

* fix: label ebook progress filters as read

* fix: show ebook related rails as book covers

* fix: remove txt ebook reader support

* fix: reject txt ebook reader files

* fix: label ebook advanced filters as read

* fix: label ebook personalized sorts as read

* fix: remove plain text reader loader path

* test: cover ebook unread catalog rules

* fix: preserve ebook reader library context

* fix: link ebook genres with library scope

* fix: encode related rail item links

* fix: encode catalog card item links

* fix: encode hero and continue item links

* fix: encode watch tonight item links

* fix: encode recommendation and search item links

* test: cover ebook scan format set

* fix: label ebook search results clearly

* fix: make global search prompt media neutral

* fix: encode catalog read API ids

* fix: encode item API ids

* fix: include ebook reader vendor in docker build

* fix: make ebook reader build clean

* fix: clean ebook embedded descriptions

* docs: plan ebook reader shell parity

* feat: add ebook reader shell controls

* fix: widen ebook scrolled reader flow

* fix: remove scrolled reader content width cap

* docs: plan ebook reader full parity

* feat: persist ebook reader config

* feat: add ebook annotations and bookmarks

* feat: add ebook reader tools and aids

* feat: add ebook advanced reader settings

* fix: keep ebook reader panel in viewport

* fix: use foliate sizing units for ebook scroll flow

* fix: keep ebook settings controls readable

* fix: simplify ebook reader settings controls

* feat(ebooks): extract local covers during scan (#98)

* feat(ebooks): extract local covers during scan

* fix(ebooks): read nullable poster paths during cover scan

* fix(catalog): coalesce nullable media artwork fields

* fix(ebooks): group sibling formats by book identity

* fix(ebooks): tolerate legacy ebook metadata encodings

* fix(ebooks): decode PDF hex metadata strings

* fix(ebooks): harden local cover extraction and format grouping

Address review findings on the local cover scan:

- Restrict generic sidecar covers (cover.jpg, folder.png, ...) to
  single-book directories, always accept images named after the book
  file, and apply exactly one cover per reconcile with sidecar taking
  precedence over the embedded cover.
- Replace the read-then-write poster update with an atomic conditional
  UPDATE (ItemRepository.SetLocalPoster) so provider/admin artwork is
  never clobbered by concurrent writers, and refresh locally owned
  posters when the extracted cover bytes change (thumbhash compare).
- Preserve UTF-8 PDF Info strings (including a UTF-8 BOM) instead of
  forcing everything through Windows-1252; the cp1252 fallback now only
  applies to non-UTF-8 bytes.
- Select EPUB covers by manifest media-type with properties="cover-image"
  outranking the EPUB2 meta name="cover" id, so XHTML cover pages no
  longer shadow the real image.
- Order CBZ pages naturally (2.jpg before 10.jpg, ch2/ before ch10/)
  when picking the cover page, via a single O(n) min-scan.
- Bump the ebook content group key scheme to version 2 and reprocess
  rows written under older versions so pre-existing libraries gain
  sibling-format grouping instead of accumulating duplicates.
- Group different formats only (a same-format sibling with colliding
  sparse metadata stays a separate item) and stop a joining sibling's
  embedded metadata from overwriting a provider-matched item.
- Decode any IANA-labelled OPF/FB2 XML charset (windows-1251, koi8-r,
  shift_jis, ...) via x/net/html/charset, and wire the charset reader
  into FB2 parsing which previously had none.
- Strip the full .fb2.zip double extension from filename-derived titles
  and group keys.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Quick <31828688+Quick104@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

* feat(ebooks): add reader profiles and ruler (#99)

* feat(ebooks): extract local covers during scan

* fix(ebooks): read nullable poster paths during cover scan

* fix(catalog): coalesce nullable media artwork fields

* fix(ebooks): group sibling formats by book identity

* fix(ebooks): tolerate legacy ebook metadata encodings

* fix(ebooks): decode PDF hex metadata strings

* feat(ebooks): add reader profiles and ruler

* fix(ebooks): address reader ruler and profile review findings

- skip renderer setStyles/render when computed styles and attributes are
  unchanged, so ruler position updates no longer re-style the book view
- drag the ruler via a local draft that commits on release, with the
  surface rect cached at pointer-down
- migrate font values persisted before the generic stacks (Inter,
  Georgia, Merriweather, legacy serif) so the font select never renders
  blank, with a Custom fallback option for unknown values
- make the ruler band click-through and move dragging to a dedicated
  keyboard-accessible slider handle so links and text selection keep
  working under the band
- share font stacks between options and profiles via READER_FONT_STACKS
- surface the active reading profile, move presets to the top of the
  settings panel, and drop the redundant profile button aria-labels

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ebooks): resolve prefer-const lint error in readest document lib

`pnpm run lint` failed on the branch because `direction` is never
reassigned in getDirection; split the destructure so only the
reassigned `writingMode` stays mutable.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Quick <31828688+Quick104@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

* Merge branch 'main' into work/ebooks-reader-base

Brings the ebook integration branch up to date with main (audiobook
library redesign, continue-watching rework and card affordances,
quic-go bump, jellycompat fixes). Conflict resolutions favor main's
generalized mechanisms and register ebooks with them:

- media scope validation goes through IsValidMediaScope (now including
  "ebook" alongside main's "video" group scope), in Go and in the web
  filter/search types
- continue-watching uses main's typed rails; reading-type sections pull
  resume points from ebook_reader_progress and the ebook library default
  section is wired to ContinueTypeConfig(ContinueTypeReading)
- item_repo keeps main's derived select-list machinery (itemColumnExpr)
  and both poster accessors (GetPoster/SetLocalPoster for ebook covers,
  GetPosterPath for audiobook covers)
- web cards/hero/watch-tonight adopt main's buildMediaPlayHref helpers,
  which now route ebooks to /reader/ebook and encode content ids;
  ebook affordances (BookOpen icon, Read verb, percent-read subtitle)
  carry over onto main's reworked components
- LibraryForm ebook support ported into main's refactored
  useLibraryForm/libraryTypes modules

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(docker): copy foliate-js vendor into Dockerfile.dev frontend stage

foliate-js is a file:vendor/foliate-js dependency, so pnpm install needs
the vendor directory before the lockfile install layer. The production
Dockerfile already copies it; the dev image was missed, breaking
make dev-deploy with ENOENT on /app/web/vendor/foliate-js.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(ebooks): render Continue Reading sections as upright poster cards

All-ebook continue sections previously fell through to the horizontal
16:9 wide card; include ebooks in the poster-variant check so book
covers render in their natural 2:3 framing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ui): stop related-rail highlight ring clipping on detail pages

Move the current-item ring onto the cover artwork with a themed
ring-offset color (matching the sidebar profile highlight) and give
the scroll container top headroom so the ring is not cut off by
overflow-x-auto. Applies to both ebook and audiobook detail rails.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(scanner): harden ebook scanning against data loss and bad metadata

- Reconcile missing ebook files like video/audio, with real per-root walk
  failure tracking (failed/unmounted roots are excluded from deletion),
  symlinked-root support via the shared logical walker, and the empty-root
  cleanup allowance before any destructive reconciliation.
- Create ebook items as 'pending' so enrichment can promote them to
  'matched' (backfill migration included), and protect matched items from
  re-scan clobbering: title/year skipped, people/series fill-empty only.
- PDF metadata: scan head + tail windows (non-linearized PDFs keep the Info
  dict at the end), require proper key delimiters, head values win.
- Cap plain .fb2 reads like .fbz entries; drop .md as an ebook format.
- gofmt internal/scanner/audiobook.go (pre-existing drift).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ebooks): make enrichment failures non-terminal with dedicated backoff state

- Provider errors now record a failure (capped retries) instead of stamping
  last_refreshed, which permanently excluded items after transient outages.
- Unconfigured metadata chains and the scan-window membership race skip the
  item without stamping or burning a retry.
- Failure tracking moves to a new ebook_enrichment_state table, decoupling
  it from media_items.refresh_failures (shared with metadata refresh debt).
- Preserve non-author people credits when persisting enrichment results.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(catalog): gate ebook progress on hidden history and centralize threshold

- Apply user_history_hidden_items gating (video semantics) to the ebook
  watched/in-progress filters, progress sort plan, and Continue Reading.
- Continue Reading pages past dismissed items via the shared collector and
  dedupes items across pages (also fixes the video path's latent exposure).
- Centralize the 0.9 finished threshold as models.EbookFinishedProgressThreshold
  with a single SQL-interpolated mirror in catalog.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(recommendations): correct watcher counting and wire ebook taste signals

- itemWatchersQuery dedupes to distinct (watcher, item) rows so one
  binge-watcher can no longer satisfy minWatchers; the eligibility floor
  now counts distinct accounts rather than profiles.
- Hidden-history gating on GetEbookReaderProgressForUser (signal reader).
- Ebook reading produces canonical implicit taste signals (weighted like
  the equivalent movie progress ratio); ebooks join taste-seed candidates.
- Stale GetRecentlyAddedItems doc comment corrected.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(api): harden ebook reader endpoints and serve a Content-Security-Policy

- Serve a CSP on all SPA HTML responses: blob/srcdoc book iframes inherit
  it, so script-src 'self' 'wasm-unsafe-eval' blocks script execution from
  malicious book content (sandbox alone is defeated by the WebKit
  allow-scripts requirement). Threat model documented on the constant.
- X-Content-Type-Options: nosniff on frontend, jellycompat, and ebook file
  responses; MIME resolution can no longer fall through to octet-stream
  for an admitted ebook file.
- Annotation PATCH: presence-aware field semantics (absent keeps, present
  sets/clears), invariant re-validation on the merged row, and an atomic
  SELECT ... FOR UPDATE read-merge-write.
- Request size caps (413) on progress/config/annotation writes;
  Content-Disposition via mime.FormatMediaType; hidden-history gating in
  the shared ebook progress lister; FK-cascade indexes for reader tables.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(api): native read-state endpoints for ebooks

- POST/DELETE /watched/{id} accepts ebook content IDs: mark read upserts
  progress 1.0 preserving the reader's file/location (or picks the
  preferred reader file for never-opened books); mark unread mirrors video
  unwatch semantics and deletes the progress row.
- /history/remove accepts ebooks: hides via user_history_hidden_items
  without touching the reading position (hidden != unread; next reading
  activity resurfaces the book, mirroring video re-watch).
- Access-filter checks match the video branch; shared logic lives in
  ebook_read_state.go. Sort metrics/user-state thresholds use the shared
  constant; profile-header fallback deduplicated.

Clients: response is {type: "ebook", affected_count: 1, played: bool};
the existing watched SSE event fires.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(web): harden the ebook reader UI

- Open-flow race: cancellation checked after every await with full stale-run
  teardown (no wrong-file progress saves, no leaked views/blob URLs);
  book.destroy() on cleanup.
- Progress: monotonic stale-response guard; visibilitychange flush uses the
  refresh-capable client, pagehide uses keepalive; per-book cross-format
  progress documented as deliberate.
- Settings: side effects out of the setState updater; local edits no longer
  clobbered by late server config; pending saves flushed on unmount/pagehide.
- TTS: generation token so Stop actually stops (Chromium/Firefox synthetic
  events); Media Session uninstalled on unmount.
- External book links: http(s) only, opened with noopener,noreferrer.
- apiBlob 512 MiB guard with a user-facing error; fraction bookmarks
  navigable; search-result key collisions fixed; dead e-ink code removed;
  getLibrarySortRelevanceScope deduplicated; md format dropped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(web): mark read/unread affordances for ebooks

- Item detail gets a Mark Read/Unread button; card menus drop the ebook
  gate and share type-aware labels/toasts (also dedupes audiobook wording).
- Watched-state invalidation includes the reader progress query key so the
  Continue button and percent refresh after toggling.
- Continue Reading dismiss copy for ebooks; dismissal path now URL-encodes
  item IDs (ebook content IDs can contain reserved characters).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* docs: record the PR #124 review and hardening pass

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-06-10 08:18:35 -04:00

1111 lines
28 KiB
Go

package scanner
import (
"archive/zip"
"bytes"
"encoding/hex"
"encoding/xml"
"fmt"
"html"
"io"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
xhtml "golang.org/x/net/html"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding/charmap"
textunicode "golang.org/x/text/encoding/unicode"
)
const maxEPUBMetadataEntrySize = 8 * 1024 * 1024
const maxPDFMetadataScanSize = 2 * 1024 * 1024
var ebookExtensions = map[string]bool{
".epub": true,
".pdf": true,
".mobi": true,
".azw": true,
".azw3": true,
".fb2": true,
".fbz": true,
".cbz": true,
".cbr": true,
}
type parsedEbook struct {
Format string
Title string
Authors []string
Description string
Publisher string
PublishedAt time.Time
Year int
Language string
ISBN string
Series string
SeriesIndex string
Genres []string
PageCount int
Cover *parsedEbookCover
}
type parsedEbookCover struct {
ContentType string
Bytes []byte
}
func SupportsEbookFile(filePath string) bool {
if strings.HasSuffix(strings.ToLower(filePath), ".fb2.zip") {
return true
}
return ebookExtensions[strings.ToLower(filepath.Ext(filePath))]
}
func parseEbookFile(path string) (book parsedEbook, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic parsing ebook %s: %v", path, r)
}
}()
format := ebookFileFormat(path)
switch format {
case ".epub":
book, err = parseEbookEPUB(path)
case ".fb2":
book, err = parseEbookFB2(path)
case ".fbz":
book, err = parseEbookFBZ(path)
case ".cbz":
book, err = parseEbookCBZ(path)
case ".pdf":
book, err = parseEbookPDF(path)
case ".mobi", ".azw", ".azw3", ".cbr":
book = parsedEbook{Format: strings.TrimPrefix(format, ".")}
default:
err = fmt.Errorf("unsupported ebook format: %s", filepath.Ext(path))
}
book.sanitize()
return book, err
}
func parseEbookPDF(path string) (parsedEbook, error) {
book := parsedEbook{Format: "pdf"}
file, err := os.Open(path)
if err != nil {
return book, err
}
defer file.Close()
head, tail, err := readPDFMetadataWindows(file)
if err != nil {
return book, err
}
info := parsePDFInfoFields(head)
// A head match comes from a linearized PDF whose Info dictionary sits at
// the start of the file and is authoritative; non-linearized PDFs (the
// common case) store the Info dictionary near the end and usually have no
// head match at all. The tail therefore only fills keys the head did not
// produce, so key-shaped byte noise inside trailing compressed streams can
// never replace a good head value.
for key, value := range parsePDFInfoFields(tail) {
if value != "" && info[key] == "" {
info[key] = value
}
}
book.Title = info["Title"]
book.Authors = splitEbookAuthors(info["Author"])
book.Description = info["Subject"]
book.Genres = splitPDFKeywords(info["Keywords"])
for _, value := range []string{
info["ISBN"],
info["Subject"],
info["Keywords"],
info["Title"],
} {
if isbn := normalizeEbookISBN(value); isbn != "" {
book.ISBN = isbn
break
}
}
if t, ok := parsePDFDate(info["CreationDate"]); ok {
book.PublishedAt = t
book.Year = t.Year()
}
return book, nil
}
// readPDFMetadataWindows reads the first and last maxPDFMetadataScanSize bytes
// of the file. The windows never overlap: for files at most one window long
// the tail is nil, and for files shorter than two windows the tail starts
// where the head ends.
func readPDFMetadataWindows(file *os.File) (head []byte, tail []byte, err error) {
head, err = io.ReadAll(io.LimitReader(file, maxPDFMetadataScanSize))
if err != nil {
return nil, nil, err
}
info, err := file.Stat()
if err != nil {
return nil, nil, err
}
size := info.Size()
if size <= int64(len(head)) {
return head, nil, nil
}
tailStart := size - maxPDFMetadataScanSize
if tailStart < int64(len(head)) {
tailStart = int64(len(head))
}
tail = make([]byte, size-tailStart)
if _, err := file.ReadAt(tail, tailStart); err != nil {
return nil, nil, err
}
return head, tail, nil
}
func ebookFileFormat(path string) string {
if strings.HasSuffix(strings.ToLower(path), ".fb2.zip") {
return ".fbz"
}
return strings.ToLower(filepath.Ext(path))
}
func (b *parsedEbook) sanitize() {
b.Format = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(b.Format)), ".")
b.Title = strings.TrimSpace(b.Title)
b.Description = cleanEbookDescription(b.Description)
b.Publisher = strings.TrimSpace(b.Publisher)
b.Language = strings.TrimSpace(b.Language)
b.ISBN = normalizeEbookISBN(b.ISBN)
b.Series = strings.TrimSpace(b.Series)
b.SeriesIndex = strings.TrimSpace(b.SeriesIndex)
b.Authors = uniqueTrimmedStrings(b.Authors)
b.Genres = uniqueTrimmedStrings(b.Genres)
if b.PageCount < 0 {
b.PageCount = 0
}
if b.Year == 0 && !b.PublishedAt.IsZero() {
b.Year = b.PublishedAt.Year()
}
}
func cleanEbookDescription(value string) string {
value = strings.TrimSpace(html.UnescapeString(value))
if value == "" {
return ""
}
if !strings.Contains(value, "<") || !strings.Contains(value, ">") {
return strings.Join(strings.Fields(value), " ")
}
tokenizer := xhtml.NewTokenizer(strings.NewReader(value))
var out strings.Builder
needsSpace := false
writeSpace := func() {
if out.Len() > 0 && !needsSpace {
needsSpace = true
}
}
for {
switch tokenizer.Next() {
case xhtml.ErrorToken:
return strings.Join(strings.Fields(out.String()), " ")
case xhtml.TextToken:
text := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text())))
if text == "" {
continue
}
if out.Len() > 0 && (needsSpace || !startsWithClosingPunctuation(text)) {
out.WriteByte(' ')
}
out.WriteString(text)
needsSpace = false
case xhtml.StartTagToken, xhtml.SelfClosingTagToken, xhtml.EndTagToken:
name, _ := tokenizer.TagName()
switch strings.ToLower(string(name)) {
case "br", "p", "div", "section", "article", "li", "ul", "ol", "blockquote", "tr":
writeSpace()
}
}
}
}
func startsWithClosingPunctuation(value string) bool {
for _, r := range value {
switch r {
case '.', ',', ';', ':', '!', '?', ')', ']', '}':
return true
default:
return false
}
}
return false
}
func looksLikeHTML(value string) bool {
value = strings.TrimSpace(value)
return strings.Contains(value, "<") && strings.Contains(value, ">")
}
func uniqueTrimmedStrings(values []string) []string {
seen := map[string]struct{}{}
var out []string
for _, value := range values {
trimmed := strings.TrimSpace(value)
key := strings.ToLower(trimmed)
if key == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, trimmed)
}
return out
}
func normalizeEbookISBN(value string) string {
value = strings.ToUpper(strings.TrimSpace(value))
for _, prefix := range []string{"ISBN-13", "ISBN-10", "ISBN"} {
if strings.HasPrefix(value, prefix) {
value = strings.TrimSpace(strings.TrimPrefix(value, prefix))
value = strings.TrimLeft(value, ": ")
break
}
}
var out strings.Builder
for _, r := range value {
if r >= '0' && r <= '9' {
out.WriteRune(r)
continue
}
if r == 'X' {
out.WriteRune(r)
}
}
candidate := out.String()
switch len(candidate) {
case 10:
if validISBN10(candidate) {
return candidate
}
case 13:
if validISBN13(candidate) {
return candidate
}
}
return ""
}
func validISBN10(candidate string) bool {
if len(candidate) != 10 {
return false
}
sum := 0
for i, r := range candidate {
value := 0
switch {
case r >= '0' && r <= '9':
value = int(r - '0')
case r == 'X' && i == 9:
value = 10
default:
return false
}
sum += value * (10 - i)
}
return sum%11 == 0
}
func validISBN13(candidate string) bool {
if len(candidate) != 13 {
return false
}
sum := 0
for i, r := range candidate {
if r < '0' || r > '9' {
return false
}
value := int(r - '0')
if i%2 == 1 {
value *= 3
}
sum += value
}
return sum%10 == 0
}
func parseEbookEPUB(path string) (parsedEbook, error) {
book := parsedEbook{Format: "epub"}
reader, err := zip.OpenReader(path)
if err != nil {
return book, err
}
defer reader.Close()
container, err := readEPUBZipEntry(&reader.Reader, "META-INF/container.xml")
if err != nil {
return book, err
}
opfPath, err := epubOPFPath(container)
if err != nil {
return book, err
}
opf, err := readEPUBZipEntry(&reader.Reader, opfPath)
if err != nil {
return book, err
}
if err := parseEPUBOPFMetadata(opf, &book); err != nil {
return book, err
}
if cover, err := extractEPUBCover(&reader.Reader, opfPath, opf); err == nil {
book.Cover = cover
}
return book, nil
}
func parseEbookFB2(path string) (parsedEbook, error) {
book := parsedEbook{Format: "fb2"}
file, err := os.Open(path)
if err != nil {
return book, err
}
defer file.Close()
// Mirror the .fbz entry cap so a plain .fb2 cannot stream unbounded
// bytes through the XML decoder.
info, err := file.Stat()
if err != nil {
return book, err
}
if info.Size() > maxEPUBMetadataEntrySize {
return book, fmt.Errorf("fb2 file too large: %s", path)
}
return parseEbookFB2Reader(io.LimitReader(file, maxEPUBMetadataEntrySize+1), "fb2")
}
func parseEbookFBZ(path string) (parsedEbook, error) {
book := parsedEbook{Format: "fbz"}
reader, err := zip.OpenReader(path)
if err != nil {
return book, err
}
defer reader.Close()
for _, file := range reader.File {
if !strings.HasSuffix(strings.ToLower(file.Name), ".fb2") {
continue
}
if file.UncompressedSize64 > maxEPUBMetadataEntrySize {
return book, fmt.Errorf("fbz entry too large: %s", file.Name)
}
entry, err := file.Open()
if err != nil {
return book, err
}
defer entry.Close()
return parseEbookFB2Reader(io.LimitReader(entry, maxEPUBMetadataEntrySize+1), "fbz")
}
return book, fmt.Errorf("fbz archive has no fb2 entry")
}
func parseEbookCBZ(path string) (parsedEbook, error) {
book := parsedEbook{Format: "cbz"}
reader, err := zip.OpenReader(path)
if err != nil {
return book, err
}
defer reader.Close()
var coverPage *zip.File
var coverKey string
for _, file := range reader.File {
if !isComicArchivePage(file.Name) {
continue
}
book.PageCount++
key := normalizedArchivePath(file.Name)
if coverPage == nil || naturalPathLess(key, coverKey) {
coverPage, coverKey = file, key
}
}
if coverPage != nil {
if cover, err := readArchiveImageCover(coverPage); err == nil {
book.Cover = cover
}
}
return book, nil
}
// naturalPathLess orders archive entry names case-insensitively with digit
// runs compared numerically, so unpadded page numbers ("2.jpg" before
// "10.jpg") and chapter directories ("ch2/" before "ch10/") sort in reading
// order instead of byte order.
func naturalPathLess(a, b string) bool {
for len(a) > 0 && len(b) > 0 {
if isASCIIDigit(a[0]) && isASCIIDigit(b[0]) {
aRun, aRest := splitDigitRun(a)
bRun, bRest := splitDigitRun(b)
aNum := strings.TrimLeft(aRun, "0")
bNum := strings.TrimLeft(bRun, "0")
if len(aNum) != len(bNum) {
return len(aNum) < len(bNum)
}
if aNum != bNum {
return aNum < bNum
}
a, b = aRest, bRest
continue
}
ar, aSize := utf8.DecodeRuneInString(a)
br, bSize := utf8.DecodeRuneInString(b)
al, bl := unicode.ToLower(ar), unicode.ToLower(br)
if al != bl {
return al < bl
}
a, b = a[aSize:], b[bSize:]
}
return len(a) < len(b)
}
func isASCIIDigit(b byte) bool {
return b >= '0' && b <= '9'
}
func splitDigitRun(s string) (run string, rest string) {
i := 0
for i < len(s) && isASCIIDigit(s[i]) {
i++
}
return s[:i], s[i:]
}
func isComicArchivePage(name string) bool {
clean := normalizedArchivePath(name)
if clean == "" || strings.HasSuffix(clean, "/") {
return false
}
base := strings.ToLower(filepath.Base(clean))
if strings.HasPrefix(base, "._") {
return false
}
parts := strings.Split(strings.ToLower(clean), "/")
for _, part := range parts {
if part == "__macosx" {
return false
}
}
switch filepath.Ext(base) {
case ".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".bmp":
return true
default:
return false
}
}
func normalizedArchivePath(name string) string {
return strings.TrimSpace(strings.ReplaceAll(name, "\\", "/"))
}
func readArchiveImageCover(file *zip.File) (*parsedEbookCover, error) {
if file == nil {
return nil, fmt.Errorf("nil archive image")
}
if file.UncompressedSize64 > maxEPUBMetadataEntrySize {
return nil, fmt.Errorf("archive cover entry too large: %s", file.Name)
}
entry, err := file.Open()
if err != nil {
return nil, err
}
defer entry.Close()
data, err := io.ReadAll(io.LimitReader(entry, maxEPUBMetadataEntrySize+1))
if err != nil {
return nil, err
}
if len(data) > maxEPUBMetadataEntrySize {
return nil, fmt.Errorf("archive cover entry too large: %s", file.Name)
}
if len(data) == 0 {
return nil, fmt.Errorf("archive cover entry empty: %s", file.Name)
}
return &parsedEbookCover{
ContentType: ebookImageContentType(file.Name),
Bytes: data,
}, nil
}
func parseEbookFB2Reader(reader io.Reader, format string) (parsedEbook, error) {
book := parsedEbook{Format: format}
var fb2 struct {
Description struct {
TitleInfo struct {
Genres []string `xml:"genre"`
Authors []struct {
FirstName string `xml:"first-name"`
MiddleName string `xml:"middle-name"`
LastName string `xml:"last-name"`
Nickname string `xml:"nickname"`
} `xml:"author"`
BookTitle string `xml:"book-title"`
Lang string `xml:"lang"`
Date struct {
Value string `xml:"value,attr"`
Text string `xml:",chardata"`
} `xml:"date"`
Sequences []struct {
Name string `xml:"name,attr"`
Number string `xml:"number,attr"`
} `xml:"sequence"`
} `xml:"title-info"`
PublishInfo struct {
ISBN string `xml:"isbn"`
Publisher string `xml:"publisher"`
Year string `xml:"year"`
} `xml:"publish-info"`
} `xml:"description"`
}
decoder := xml.NewDecoder(reader)
decoder.CharsetReader = ebookXMLCharsetReader
if err := decoder.Decode(&fb2); err != nil {
return book, err
}
book.Title = fb2.Description.TitleInfo.BookTitle
book.Language = fb2.Description.TitleInfo.Lang
book.Genres = fb2.Description.TitleInfo.Genres
if t, ok := parseEbookDate(firstNonEmpty(fb2.Description.TitleInfo.Date.Value, fb2.Description.TitleInfo.Date.Text)); ok {
book.PublishedAt = t
}
for _, author := range fb2.Description.TitleInfo.Authors {
name := strings.Join(uniqueTrimmedStrings([]string{
author.FirstName,
author.MiddleName,
author.LastName,
}), " ")
if name == "" {
name = author.Nickname
}
book.Authors = append(book.Authors, name)
}
if len(fb2.Description.TitleInfo.Sequences) > 0 {
book.Series = fb2.Description.TitleInfo.Sequences[0].Name
book.SeriesIndex = fb2.Description.TitleInfo.Sequences[0].Number
}
book.ISBN = fb2.Description.PublishInfo.ISBN
book.Publisher = fb2.Description.PublishInfo.Publisher
if year, err := strconv.Atoi(strings.TrimSpace(fb2.Description.PublishInfo.Year)); err == nil {
book.Year = year
}
return book, nil
}
func parsePDFInfoFields(data []byte) map[string]string {
fields := map[string]string{}
for _, key := range []string{"Title", "Author", "Subject", "Keywords", "CreationDate", "ISBN"} {
if value, ok := findPDFInfoValue(data, key); ok {
fields[key] = value
}
}
return fields
}
// pdfWhitespace is the PDF whitespace character set (ISO 32000-1, table 1).
const pdfWhitespace = "\x00\t\n\f\r "
// isPDFTokenDelimiter reports whether b legally terminates a PDF name token.
// Without this check a key with a shared prefix (e.g. "/TitleSort") would be
// mistaken for the key itself (e.g. "/Title").
func isPDFTokenDelimiter(b byte) bool {
switch b {
case '\x00', '\t', '\n', '\f', '\r', ' ', '(', ')', '<', '>', '[', ']', '{', '}', '/', '%':
return true
default:
return false
}
}
// findPDFInfoValue scans every occurrence of "/<key>" in the window and
// returns the first whose token is properly delimited and whose value parses
// as a PDF string. Raw byte search can match key-shaped noise inside
// compressed streams, so a failed parse moves on to the next occurrence
// instead of giving up.
func findPDFInfoValue(data []byte, key string) (string, bool) {
token := []byte("/" + key)
for offset := 0; offset < len(data); {
idx := bytes.Index(data[offset:], token)
if idx < 0 {
return "", false
}
idx += offset
offset = idx + len(token)
rest := data[idx+len(token):]
if len(rest) == 0 {
return "", false
}
if !isPDFTokenDelimiter(rest[0]) {
continue
}
trimmed := bytes.TrimLeft(rest, pdfWhitespace)
if len(trimmed) == 0 {
return "", false
}
if value, ok := readPDFString(trimmed); ok {
return value, true
}
}
return "", false
}
func decodeEbookXML(data []byte, v any) error {
data = normalizeEbookXMLVersion(data)
decoder := xml.NewDecoder(bytes.NewReader(data))
decoder.CharsetReader = ebookXMLCharsetReader
return decoder.Decode(v)
}
func normalizeEbookXMLVersion(data []byte) []byte {
for _, needle := range []string{`version="1.1"`, `version='1.1'`} {
idx := bytes.Index(data, []byte(needle))
if idx < 0 || idx > 128 {
continue
}
out := append([]byte(nil), data...)
copy(out[idx:], strings.Replace(needle, "1.1", "1.0", 1))
return out
}
return data
}
func ebookXMLCharsetReader(label string, input io.Reader) (io.Reader, error) {
name := strings.ToLower(strings.TrimSpace(label))
if name == "" || name == "utf-8" || name == "utf8" {
return input, nil
}
reader, err := charset.NewReaderLabel(name, input)
if err != nil {
return nil, fmt.Errorf("unsupported ebook XML encoding %q: %w", label, err)
}
return reader, nil
}
func readPDFString(data []byte) (string, bool) {
switch data[0] {
case '(':
return readPDFLiteralString(data)
case '<':
if len(data) > 1 && data[1] == '<' {
return "", false
}
return readPDFHexString(data)
default:
return "", false
}
}
func readPDFLiteralString(data []byte) (string, bool) {
if len(data) == 0 || data[0] != '(' {
return "", false
}
var out []byte
depth := 1
escaped := false
for _, b := range data[1:] {
if escaped {
switch b {
case 'n':
out = append(out, '\n')
case 'r':
out = append(out, '\r')
case 't':
out = append(out, '\t')
case 'b':
out = append(out, '\b')
case 'f':
out = append(out, '\f')
default:
out = append(out, b)
}
escaped = false
continue
}
switch b {
case '\\':
escaped = true
case '(':
depth++
out = append(out, b)
case ')':
depth--
if depth == 0 {
return strings.TrimSpace(decodePDFLiteralBytes(out)), true
}
out = append(out, b)
default:
out = append(out, b)
}
}
return "", false
}
func readPDFHexString(data []byte) (string, bool) {
if len(data) == 0 || data[0] != '<' {
return "", false
}
end := bytes.IndexByte(data[1:], '>')
if end < 0 {
return "", false
}
raw := data[1 : end+1]
var cleaned []byte
for _, b := range raw {
switch {
case b == ' ' || b == '\t' || b == '\r' || b == '\n':
continue
case (b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F'):
cleaned = append(cleaned, b)
default:
return "", false
}
}
if len(cleaned)%2 == 1 {
cleaned = append(cleaned, '0')
}
decoded := make([]byte, hex.DecodedLen(len(cleaned)))
if _, err := hex.Decode(decoded, cleaned); err != nil {
return "", false
}
return strings.TrimSpace(decodePDFLiteralBytes(decoded)), true
}
func decodePDFLiteralBytes(data []byte) string {
if len(data) == 0 {
return ""
}
switch {
case bytes.HasPrefix(data, []byte{0xfe, 0xff}):
if decoded, err := textunicode.UTF16(textunicode.BigEndian, textunicode.ExpectBOM).NewDecoder().Bytes(data); err == nil {
return string(decoded)
}
case bytes.HasPrefix(data, []byte{0xff, 0xfe}):
if decoded, err := textunicode.UTF16(textunicode.LittleEndian, textunicode.ExpectBOM).NewDecoder().Bytes(data); err == nil {
return string(decoded)
}
}
// The PDF spec says non-UTF-16 strings are PDFDocEncoding, but real-world
// producers commonly emit UTF-8 (PDF 2.0 even allows a UTF-8 BOM). Only
// fall back to the Windows-1252 approximation for non-UTF-8 bytes so
// UTF-8 metadata is not mojibaked.
data = bytes.TrimPrefix(data, []byte{0xef, 0xbb, 0xbf})
if utf8.Valid(data) {
return string(data)
}
if decoded, err := charmap.Windows1252.NewDecoder().Bytes(data); err == nil {
return string(decoded)
}
return strings.ToValidUTF8(string(data), "")
}
func splitEbookAuthors(value string) []string {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
parts := strings.FieldsFunc(value, func(r rune) bool {
return r == ';' || r == '|'
})
if len(parts) == 1 {
parts = strings.Split(value, " and ")
}
return uniqueTrimmedStrings(parts)
}
func splitPDFKeywords(value string) []string {
if strings.TrimSpace(value) == "" {
return nil
}
return uniqueTrimmedStrings(strings.FieldsFunc(value, func(r rune) bool {
return r == ',' || r == ';'
}))
}
func parsePDFDate(value string) (time.Time, bool) {
value = strings.TrimSpace(value)
if strings.HasPrefix(value, "D:") {
value = strings.TrimPrefix(value, "D:")
}
value = strings.TrimSuffix(value, "Z")
if len(value) >= 14 {
if t, err := time.Parse("20060102150405", value[:14]); err == nil && t.Year() > 0 {
return t, true
}
}
if len(value) >= 8 {
if t, err := time.Parse("20060102", value[:8]); err == nil && t.Year() > 0 {
return t, true
}
}
if len(value) >= 4 {
if t, err := time.Parse("2006", value[:4]); err == nil && t.Year() > 0 {
return t, true
}
}
return time.Time{}, false
}
func readEPUBZipEntry(reader *zip.Reader, name string) ([]byte, error) {
for _, file := range reader.File {
if file.Name != name {
continue
}
entry, err := file.Open()
if err != nil {
return nil, err
}
defer entry.Close()
limited := io.LimitReader(entry, maxEPUBMetadataEntrySize+1)
data, err := io.ReadAll(limited)
if err != nil {
return nil, err
}
if len(data) > maxEPUBMetadataEntrySize {
return nil, fmt.Errorf("epub entry too large: %s", name)
}
return data, nil
}
return nil, fmt.Errorf("epub entry not found: %s", name)
}
func epubOPFPath(container []byte) (string, error) {
var parsed struct {
Rootfiles []struct {
FullPath string `xml:"full-path,attr"`
} `xml:"rootfiles>rootfile"`
}
if err := decodeEbookXML(container, &parsed); err != nil {
return "", err
}
for _, rootfile := range parsed.Rootfiles {
if strings.TrimSpace(rootfile.FullPath) != "" {
return rootfile.FullPath, nil
}
}
return "", fmt.Errorf("epub container has no rootfile")
}
func parseEPUBOPFMetadata(opf []byte, book *parsedEbook) error {
var parsed struct {
Metadata struct {
Titles []string `xml:"title"`
Creators []string `xml:"creator"`
Identifiers []string `xml:"identifier"`
Publisher string `xml:"publisher"`
Dates []string `xml:"date"`
Language string `xml:"language"`
Subjects []string `xml:"subject"`
Descriptions []string `xml:"description"`
Meta []struct {
Name string `xml:"name,attr"`
Property string `xml:"property,attr"`
Content string `xml:"content,attr"`
Value string `xml:",chardata"`
} `xml:"meta"`
} `xml:"metadata"`
}
if err := decodeEbookXML(opf, &parsed); err != nil {
return err
}
book.Title = firstNonEmpty(parsed.Metadata.Titles...)
book.Authors = append(book.Authors, parsed.Metadata.Creators...)
book.Publisher = parsed.Metadata.Publisher
book.Language = parsed.Metadata.Language
book.Genres = append(book.Genres, parsed.Metadata.Subjects...)
book.Description = firstNonEmpty(parsed.Metadata.Descriptions...)
for _, identifier := range parsed.Metadata.Identifiers {
if isbn := normalizeEbookISBN(identifier); isbn != "" {
book.ISBN = isbn
break
}
}
for _, date := range parsed.Metadata.Dates {
if t, ok := parseEbookDate(date); ok {
book.PublishedAt = t
book.Year = t.Year()
break
}
}
for _, meta := range parsed.Metadata.Meta {
name := strings.ToLower(strings.TrimSpace(firstNonEmpty(meta.Name, meta.Property)))
value := strings.TrimSpace(firstNonEmpty(meta.Content, meta.Value))
switch name {
case "calibre:series", "belongs-to-collection":
book.Series = value
case "calibre:series_index", "group-position":
book.SeriesIndex = value
case "calibre:isbn", "isbn", "schema:isbn":
if book.ISBN == "" {
book.ISBN = normalizeEbookISBN(value)
}
}
}
return nil
}
func extractEPUBCover(reader *zip.Reader, opfPath string, opf []byte) (*parsedEbookCover, error) {
var parsed struct {
Metadata struct {
Meta []struct {
Name string `xml:"name,attr"`
Content string `xml:"content,attr"`
} `xml:"meta"`
} `xml:"metadata"`
Manifest struct {
Items []struct {
ID string `xml:"id,attr"`
Href string `xml:"href,attr"`
MediaType string `xml:"media-type,attr"`
Properties string `xml:"properties,attr"`
} `xml:"item"`
} `xml:"manifest"`
}
if err := decodeEbookXML(opf, &parsed); err != nil {
return nil, err
}
coverID := ""
for _, meta := range parsed.Metadata.Meta {
if strings.EqualFold(strings.TrimSpace(meta.Name), "cover") {
coverID = strings.TrimSpace(meta.Content)
break
}
}
// An EPUB3 properties="cover-image" item is authoritative; the EPUB2
// <meta name="cover"> id frequently points at the XHTML cover *page*
// rather than the image, so it ranks lower and non-image manifest items
// are skipped entirely instead of shadowing a later real cover image.
var coverHref string
var coverType string
coverRank := 0
for _, item := range parsed.Manifest.Items {
rank := 0
for _, prop := range strings.Fields(strings.ToLower(item.Properties)) {
if prop == "cover-image" {
rank = 2
break
}
}
if rank == 0 && coverID != "" && strings.TrimSpace(item.ID) == coverID {
rank = 1
}
if rank <= coverRank || !isEPUBImageManifestItem(item.MediaType, item.Href) {
continue
}
coverHref = strings.TrimSpace(item.Href)
coverType = strings.TrimSpace(item.MediaType)
coverRank = rank
if coverRank == 2 {
break
}
}
if coverHref == "" {
return nil, fmt.Errorf("epub cover not referenced")
}
coverPath := resolveEPUBRelativePath(opfPath, coverHref)
data, err := readEPUBZipEntry(reader, coverPath)
if err != nil {
return nil, err
}
if coverType == "" {
coverType = ebookImageContentType(coverPath)
}
return &parsedEbookCover{ContentType: coverType, Bytes: data}, nil
}
func isEPUBImageManifestItem(mediaType, href string) bool {
mt := strings.ToLower(strings.TrimSpace(mediaType))
if mt != "" {
return strings.HasPrefix(mt, "image/")
}
return ebookImageContentType(href) != "application/octet-stream"
}
func resolveEPUBRelativePath(baseFile string, href string) string {
cleanHref := strings.TrimSpace(href)
if decoded, err := urlPathUnescape(cleanHref); err == nil {
cleanHref = decoded
}
baseDir := path.Dir(strings.ReplaceAll(baseFile, "\\", "/"))
if baseDir == "." || strings.HasPrefix(cleanHref, "/") {
baseDir = ""
}
return strings.TrimPrefix(path.Clean(path.Join(baseDir, cleanHref)), "/")
}
func urlPathUnescape(value string) (string, error) {
var out strings.Builder
for i := 0; i < len(value); i++ {
if value[i] != '%' || i+2 >= len(value) {
out.WriteByte(value[i])
continue
}
hi := fromHex(value[i+1])
lo := fromHex(value[i+2])
if hi < 0 || lo < 0 {
out.WriteByte(value[i])
continue
}
out.WriteByte(byte(hi<<4 | lo))
i += 2
}
return out.String(), nil
}
func fromHex(b byte) int {
switch {
case b >= '0' && b <= '9':
return int(b - '0')
case b >= 'a' && b <= 'f':
return int(b-'a') + 10
case b >= 'A' && b <= 'F':
return int(b-'A') + 10
default:
return -1
}
}
func ebookImageContentType(name string) string {
switch strings.ToLower(filepath.Ext(name)) {
case ".jpg", ".jpeg":
return "image/jpeg"
case ".png":
return "image/png"
case ".webp":
return "image/webp"
case ".gif":
return "image/gif"
case ".avif":
return "image/avif"
case ".bmp":
return "image/bmp"
default:
return "application/octet-stream"
}
}
func parseEbookDate(value string) (time.Time, bool) {
value = strings.TrimSpace(value)
for _, layout := range []string{"2006-01-02", "2006-01", "2006"} {
if t, err := time.Parse(layout, value); err == nil {
return t, true
}
}
return time.Time{}, false
}