Files
silo-server/internal/recommendations/repo.go
T
8b70357703 feat(ebooks): first-class ebook libraries, scanner, and reader (#124)
* docs: define ebook architecture matching audiobooks

* docs: plan ebook audiobook-parity implementation

* feat: add ebook scanner parser foundation

* fix: harden ebook scanner foundation

* fix: handle ebook isbn labels

* fix: guard ebook subtree scans

* feat: scan ebook libraries in core

* fix: preserve ebook scan people credits

* fix: refresh ebook scan metadata safely

* feat: persist ebook series membership

* test: cover ebook series persistence decisions

* fix: address ebook scanner PR review

* docs: clarify ebook foundation PR scope

* feat: add ebook metadata enricher

* fix: harden ebook poster cache

* feat: wire ebook metadata sync task

* feat: expose ebook library metadata setup

* feat: add ebook catalog scope support

* feat: add ebook detail view

* feat: label ebook file versions by format

* feat: use file-size copy for downloads

* feat: use file language in download dialog

* test: cover ebook detail authors and downloads

* fix: drop narrator credits from ebook scanner merges

* fix: align ebook collection filters with book media

* fix: drop asin provider ids from ebook enrichment

* fix: force ebook people refresh for stale narrators

* chore: omit ebook planning docs from branch

* feat: add ebook detail related content

* feat: add ebook reader file entrypoint

* feat: render ebooks with foliate reader

* feat: persist ebook reader progress

* feat: add ebook reader controls

* feat: extract ebook pdf metadata

* feat: favor scanner isbn during ebook enrichment

* feat: extract fbz ebook metadata

* feat: count cbz ebook pages

* feat: show ebook file page counts

* feat: show ebook download summaries

* feat: switch ebook reader files

* feat: prefer epub for ebook read action

* feat: surface ebook reader progress

* feat: sync ebook reader progress cache

* feat: hide ebook read action for unsupported files

* feat: filter ebook reader file selector

* fix: serve fbz ebook archives with reader mime type

* fix: detect fbz ebooks from compound filename

* fix: authorize fbz ebooks from compound filename

* fix: scope ebook catalog facets

* fix: reject narrator queries for ebooks

* fix: build ebook recommendation text from authors

* fix: include ebooks in embedding eligibility

* fix: include ebooks in recommendation media mix

* fix: include ebooks in recently added recommendations

* feat: include ebook progress in recommendation signals

* feat: include ebooks in continue watching sections

* feat: include ebooks in catalog progress metrics

* fix: read ebook isbn from epub metadata

* fix: filter ebook asin provider aliases

* fix: fall back from unsupported ebook reader files

* fix: sort ebook catalogs by reader progress

* fix: filter ebook catalogs by reader progress

* fix: include ebooks in last watched catalog filters

* feat: reflect ebook reader progress in item user state

* feat: share ebook progress state across item surfaces

* feat: report ebook scan progress

* fix: include ebook activity in recommendations

* fix: expose ebook reader progress on item detail

* fix: support ebook subtree scans

* fix: honor profile header for ebook item progress

* fix: add ebook library default sections

* fix: route ebook continue cards to reader

* fix: hide watched toggle for ebooks

* fix: route ebook watch tonight cards to reader

* fix: route ebook hero actions to reader

* fix: detect archive ebook reader formats by filename

* feat: cache embedded ebook covers during scan

* fix: encode ebook hero reader links

* fix: persist non-epub ebook reader progress

* fix: scope narrator catalog badges to audiobooks

* fix: merge ebook reader progress during item repair

* fix: label ebook progress filters as read

* fix: show ebook related rails as book covers

* fix: remove txt ebook reader support

* fix: reject txt ebook reader files

* fix: label ebook advanced filters as read

* fix: label ebook personalized sorts as read

* fix: remove plain text reader loader path

* test: cover ebook unread catalog rules

* fix: preserve ebook reader library context

* fix: link ebook genres with library scope

* fix: encode related rail item links

* fix: encode catalog card item links

* fix: encode hero and continue item links

* fix: encode watch tonight item links

* fix: encode recommendation and search item links

* test: cover ebook scan format set

* fix: label ebook search results clearly

* fix: make global search prompt media neutral

* fix: encode catalog read API ids

* fix: encode item API ids

* fix: include ebook reader vendor in docker build

* fix: make ebook reader build clean

* fix: clean ebook embedded descriptions

* docs: plan ebook reader shell parity

* feat: add ebook reader shell controls

* fix: widen ebook scrolled reader flow

* fix: remove scrolled reader content width cap

* docs: plan ebook reader full parity

* feat: persist ebook reader config

* feat: add ebook annotations and bookmarks

* feat: add ebook reader tools and aids

* feat: add ebook advanced reader settings

* fix: keep ebook reader panel in viewport

* fix: use foliate sizing units for ebook scroll flow

* fix: keep ebook settings controls readable

* fix: simplify ebook reader settings controls

* feat(ebooks): extract local covers during scan (#98)

* feat(ebooks): extract local covers during scan

* fix(ebooks): read nullable poster paths during cover scan

* fix(catalog): coalesce nullable media artwork fields

* fix(ebooks): group sibling formats by book identity

* fix(ebooks): tolerate legacy ebook metadata encodings

* fix(ebooks): decode PDF hex metadata strings

* fix(ebooks): harden local cover extraction and format grouping

Address review findings on the local cover scan:

- Restrict generic sidecar covers (cover.jpg, folder.png, ...) to
  single-book directories, always accept images named after the book
  file, and apply exactly one cover per reconcile with sidecar taking
  precedence over the embedded cover.
- Replace the read-then-write poster update with an atomic conditional
  UPDATE (ItemRepository.SetLocalPoster) so provider/admin artwork is
  never clobbered by concurrent writers, and refresh locally owned
  posters when the extracted cover bytes change (thumbhash compare).
- Preserve UTF-8 PDF Info strings (including a UTF-8 BOM) instead of
  forcing everything through Windows-1252; the cp1252 fallback now only
  applies to non-UTF-8 bytes.
- Select EPUB covers by manifest media-type with properties="cover-image"
  outranking the EPUB2 meta name="cover" id, so XHTML cover pages no
  longer shadow the real image.
- Order CBZ pages naturally (2.jpg before 10.jpg, ch2/ before ch10/)
  when picking the cover page, via a single O(n) min-scan.
- Bump the ebook content group key scheme to version 2 and reprocess
  rows written under older versions so pre-existing libraries gain
  sibling-format grouping instead of accumulating duplicates.
- Group different formats only (a same-format sibling with colliding
  sparse metadata stays a separate item) and stop a joining sibling's
  embedded metadata from overwriting a provider-matched item.
- Decode any IANA-labelled OPF/FB2 XML charset (windows-1251, koi8-r,
  shift_jis, ...) via x/net/html/charset, and wire the charset reader
  into FB2 parsing which previously had none.
- Strip the full .fb2.zip double extension from filename-derived titles
  and group keys.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Quick <31828688+Quick104@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

* feat(ebooks): add reader profiles and ruler (#99)

* feat(ebooks): extract local covers during scan

* fix(ebooks): read nullable poster paths during cover scan

* fix(catalog): coalesce nullable media artwork fields

* fix(ebooks): group sibling formats by book identity

* fix(ebooks): tolerate legacy ebook metadata encodings

* fix(ebooks): decode PDF hex metadata strings

* feat(ebooks): add reader profiles and ruler

* fix(ebooks): address reader ruler and profile review findings

- skip renderer setStyles/render when computed styles and attributes are
  unchanged, so ruler position updates no longer re-style the book view
- drag the ruler via a local draft that commits on release, with the
  surface rect cached at pointer-down
- migrate font values persisted before the generic stacks (Inter,
  Georgia, Merriweather, legacy serif) so the font select never renders
  blank, with a Custom fallback option for unknown values
- make the ruler band click-through and move dragging to a dedicated
  keyboard-accessible slider handle so links and text selection keep
  working under the band
- share font stacks between options and profiles via READER_FONT_STACKS
- surface the active reading profile, move presets to the top of the
  settings panel, and drop the redundant profile button aria-labels

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ebooks): resolve prefer-const lint error in readest document lib

`pnpm run lint` failed on the branch because `direction` is never
reassigned in getDirection; split the destructure so only the
reassigned `writingMode` stays mutable.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Quick <31828688+Quick104@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

* Merge branch 'main' into work/ebooks-reader-base

Brings the ebook integration branch up to date with main (audiobook
library redesign, continue-watching rework and card affordances,
quic-go bump, jellycompat fixes). Conflict resolutions favor main's
generalized mechanisms and register ebooks with them:

- media scope validation goes through IsValidMediaScope (now including
  "ebook" alongside main's "video" group scope), in Go and in the web
  filter/search types
- continue-watching uses main's typed rails; reading-type sections pull
  resume points from ebook_reader_progress and the ebook library default
  section is wired to ContinueTypeConfig(ContinueTypeReading)
- item_repo keeps main's derived select-list machinery (itemColumnExpr)
  and both poster accessors (GetPoster/SetLocalPoster for ebook covers,
  GetPosterPath for audiobook covers)
- web cards/hero/watch-tonight adopt main's buildMediaPlayHref helpers,
  which now route ebooks to /reader/ebook and encode content ids;
  ebook affordances (BookOpen icon, Read verb, percent-read subtitle)
  carry over onto main's reworked components
- LibraryForm ebook support ported into main's refactored
  useLibraryForm/libraryTypes modules

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(docker): copy foliate-js vendor into Dockerfile.dev frontend stage

foliate-js is a file:vendor/foliate-js dependency, so pnpm install needs
the vendor directory before the lockfile install layer. The production
Dockerfile already copies it; the dev image was missed, breaking
make dev-deploy with ENOENT on /app/web/vendor/foliate-js.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(ebooks): render Continue Reading sections as upright poster cards

All-ebook continue sections previously fell through to the horizontal
16:9 wide card; include ebooks in the poster-variant check so book
covers render in their natural 2:3 framing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ui): stop related-rail highlight ring clipping on detail pages

Move the current-item ring onto the cover artwork with a themed
ring-offset color (matching the sidebar profile highlight) and give
the scroll container top headroom so the ring is not cut off by
overflow-x-auto. Applies to both ebook and audiobook detail rails.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(scanner): harden ebook scanning against data loss and bad metadata

- Reconcile missing ebook files like video/audio, with real per-root walk
  failure tracking (failed/unmounted roots are excluded from deletion),
  symlinked-root support via the shared logical walker, and the empty-root
  cleanup allowance before any destructive reconciliation.
- Create ebook items as 'pending' so enrichment can promote them to
  'matched' (backfill migration included), and protect matched items from
  re-scan clobbering: title/year skipped, people/series fill-empty only.
- PDF metadata: scan head + tail windows (non-linearized PDFs keep the Info
  dict at the end), require proper key delimiters, head values win.
- Cap plain .fb2 reads like .fbz entries; drop .md as an ebook format.
- gofmt internal/scanner/audiobook.go (pre-existing drift).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(ebooks): make enrichment failures non-terminal with dedicated backoff state

- Provider errors now record a failure (capped retries) instead of stamping
  last_refreshed, which permanently excluded items after transient outages.
- Unconfigured metadata chains and the scan-window membership race skip the
  item without stamping or burning a retry.
- Failure tracking moves to a new ebook_enrichment_state table, decoupling
  it from media_items.refresh_failures (shared with metadata refresh debt).
- Preserve non-author people credits when persisting enrichment results.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(catalog): gate ebook progress on hidden history and centralize threshold

- Apply user_history_hidden_items gating (video semantics) to the ebook
  watched/in-progress filters, progress sort plan, and Continue Reading.
- Continue Reading pages past dismissed items via the shared collector and
  dedupes items across pages (also fixes the video path's latent exposure).
- Centralize the 0.9 finished threshold as models.EbookFinishedProgressThreshold
  with a single SQL-interpolated mirror in catalog.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(recommendations): correct watcher counting and wire ebook taste signals

- itemWatchersQuery dedupes to distinct (watcher, item) rows so one
  binge-watcher can no longer satisfy minWatchers; the eligibility floor
  now counts distinct accounts rather than profiles.
- Hidden-history gating on GetEbookReaderProgressForUser (signal reader).
- Ebook reading produces canonical implicit taste signals (weighted like
  the equivalent movie progress ratio); ebooks join taste-seed candidates.
- Stale GetRecentlyAddedItems doc comment corrected.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(api): harden ebook reader endpoints and serve a Content-Security-Policy

- Serve a CSP on all SPA HTML responses: blob/srcdoc book iframes inherit
  it, so script-src 'self' 'wasm-unsafe-eval' blocks script execution from
  malicious book content (sandbox alone is defeated by the WebKit
  allow-scripts requirement). Threat model documented on the constant.
- X-Content-Type-Options: nosniff on frontend, jellycompat, and ebook file
  responses; MIME resolution can no longer fall through to octet-stream
  for an admitted ebook file.
- Annotation PATCH: presence-aware field semantics (absent keeps, present
  sets/clears), invariant re-validation on the merged row, and an atomic
  SELECT ... FOR UPDATE read-merge-write.
- Request size caps (413) on progress/config/annotation writes;
  Content-Disposition via mime.FormatMediaType; hidden-history gating in
  the shared ebook progress lister; FK-cascade indexes for reader tables.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(api): native read-state endpoints for ebooks

- POST/DELETE /watched/{id} accepts ebook content IDs: mark read upserts
  progress 1.0 preserving the reader's file/location (or picks the
  preferred reader file for never-opened books); mark unread mirrors video
  unwatch semantics and deletes the progress row.
- /history/remove accepts ebooks: hides via user_history_hidden_items
  without touching the reading position (hidden != unread; next reading
  activity resurfaces the book, mirroring video re-watch).
- Access-filter checks match the video branch; shared logic lives in
  ebook_read_state.go. Sort metrics/user-state thresholds use the shared
  constant; profile-header fallback deduplicated.

Clients: response is {type: "ebook", affected_count: 1, played: bool};
the existing watched SSE event fires.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(web): harden the ebook reader UI

- Open-flow race: cancellation checked after every await with full stale-run
  teardown (no wrong-file progress saves, no leaked views/blob URLs);
  book.destroy() on cleanup.
- Progress: monotonic stale-response guard; visibilitychange flush uses the
  refresh-capable client, pagehide uses keepalive; per-book cross-format
  progress documented as deliberate.
- Settings: side effects out of the setState updater; local edits no longer
  clobbered by late server config; pending saves flushed on unmount/pagehide.
- TTS: generation token so Stop actually stops (Chromium/Firefox synthetic
  events); Media Session uninstalled on unmount.
- External book links: http(s) only, opened with noopener,noreferrer.
- apiBlob 512 MiB guard with a user-facing error; fraction bookmarks
  navigable; search-result key collisions fixed; dead e-ink code removed;
  getLibrarySortRelevanceScope deduplicated; md format dropped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* feat(web): mark read/unread affordances for ebooks

- Item detail gets a Mark Read/Unread button; card menus drop the ebook
  gate and share type-aware labels/toasts (also dedupes audiobook wording).
- Watched-state invalidation includes the reader progress query key so the
  Continue button and percent refresh after toggling.
- Continue Reading dismiss copy for ebooks; dismissal path now URL-encodes
  item IDs (ebook content IDs can contain reserved characters).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* docs: record the PR #124 review and hardening pass

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-06-10 08:18:35 -04:00

1947 lines
63 KiB
Go

package recommendations
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"github.com/Silo-Server/silo-server/internal/access"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/userstore"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/pgvector/pgvector-go"
)
// padToLength zero-pads a vector to the target dimension. If the vector is
// already at or above the target length, it is returned as-is. This allows
// local models producing shorter vectors (e.g. 768-dim) to be stored in a
// fixed-dimension column (3072) without schema changes.
func padToLength(vec []float32, dim int) []float32 {
if len(vec) >= dim {
return vec
}
padded := make([]float32, dim)
copy(padded, vec)
return padded
}
func ensureCanonicalDimensions(vec []float32) ([]float32, error) {
if len(vec) > CanonicalEmbeddingDimensions {
return nil, fmt.Errorf("embedding vector length %d exceeds canonical dimension %d", len(vec), CanonicalEmbeddingDimensions)
}
return padToLength(vec, CanonicalEmbeddingDimensions), nil
}
const embeddingLockSettingKey = "recommendations.embedding_lock"
const minHNSWEfSearch = 200
// watchedActivityCTE unifies video watch progress and ebook reader progress
// into one activity stream. Episodes roll up to their parent series via
// COALESCE(e.series_id, ...). The ebook completed flag is derived from the
// shared finished-progress threshold.
var watchedActivityCTE = fmt.Sprintf(`
watched_activity AS (
SELECT COALESCE(e.series_id, wp.media_item_id) AS item_id,
wp.media_item_id AS leaf_item_id,
wp.user_id,
wp.profile_id,
wp.user_id::text || ':' || COALESCE(wp.profile_id, '') AS watcher_id,
wp.updated_at,
wp.completed AS completed
FROM user_watch_progress wp
LEFT JOIN episodes e ON e.content_id = wp.media_item_id
WHERE (wp.completed = true OR (wp.duration_seconds > 0 AND wp.position_seconds / wp.duration_seconds >= 0.5))
AND NOT EXISTS (
SELECT 1
FROM user_history_hidden_items hhi
WHERE hhi.user_id = wp.user_id
AND hhi.profile_id = wp.profile_id
AND hhi.media_item_id = wp.media_item_id
AND wp.updated_at <= hhi.hidden_before
)
UNION ALL
SELECT erp.content_id AS item_id,
erp.content_id AS leaf_item_id,
erp.user_id,
erp.profile_id,
erp.user_id::text || ':' || COALESCE(erp.profile_id, '') AS watcher_id,
erp.updated_at,
erp.progress >= %s AS completed
FROM ebook_reader_progress erp
WHERE erp.progress >= 0.5
AND NOT EXISTS (
SELECT 1
FROM user_history_hidden_items hhi
WHERE hhi.user_id = erp.user_id
AND hhi.profile_id = erp.profile_id
AND hhi.media_item_id = erp.content_id
AND erp.updated_at <= hhi.hidden_before
)
)`, catalog.EbookFinishedProgressThresholdSQL)
var tasteSeedCandidateQuery = fmt.Sprintf(`
WITH %s,
watched_counts AS (
SELECT item_id, COUNT(DISTINCT watcher_id) AS watch_count
FROM watched_activity
WHERE updated_at > NOW() - INTERVAL '180 days'
GROUP BY item_id
)
SELECT mi.content_id
FROM media_items mi
LEFT JOIN watched_counts wc ON wc.item_id = mi.content_id
WHERE %s
AND mi.type IN ('movie', 'series', 'audiobook', 'ebook')
AND mi.poster_path IS NOT NULL
AND mi.poster_path <> ''
ORDER BY COALESCE(wc.watch_count, 0) DESC,
CASE
WHEN mi.rating_imdb IS NOT NULL THEN 2
WHEN mi.rating_tmdb IS NOT NULL AND mi.rating_tmdb < 9.5 THEN 1
ELSE 0
END DESC,
mi.rating_imdb DESC NULLS LAST,
CASE WHEN mi.rating_tmdb < 9.5 THEN mi.rating_tmdb END DESC NULLS LAST,
mi.year DESC NULLS LAST,
mi.content_id ASC
LIMIT $1 OFFSET $2`, watchedActivityCTE, recommendationItemEligibilityWhereClause("mi"))
// Repo provides database operations for the recommendation system.
type Repo struct {
pool *pgxpool.Pool
}
// NewRepo creates a new Repo with the given connection pool.
func NewRepo(pool *pgxpool.Pool) *Repo {
return &Repo{pool: pool}
}
func hnswEfSearch(candidateLimit int) int {
return max(candidateLimit, minHNSWEfSearch)
}
func (r *Repo) withHNSWCandidateScan(ctx context.Context, candidateLimit int, fn func(pgx.Tx) error) error {
tx, err := r.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return fmt.Errorf("begin hnsw candidate scan tx: %w", err)
}
defer tx.Rollback(ctx)
_, err = tx.Exec(ctx, `
SELECT set_config('hnsw.iterative_scan', 'relaxed_order', true),
set_config('hnsw.ef_search', $1, true)
`, fmt.Sprintf("%d", hnswEfSearch(candidateLimit)))
if err != nil {
return fmt.Errorf("configure hnsw candidate scan: %w", err)
}
if err := fn(tx); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit hnsw candidate scan tx: %w", err)
}
return nil
}
// UserSimilarity represents a user with a similarity score relative to the
// requesting user's taste profile.
type UserSimilarity struct {
UserID int
ProfileID string
Score float64
}
type EmbeddingTextCandidate struct {
MediaItemID string
Model string
CanonicalText string
}
func embeddingEligibilityWhereClause() string {
return recommendationItemEligibilityWhereClause("mi")
}
func recommendationItemEligibilityWhereClause(alias string) string {
alias = strings.TrimSpace(alias)
if alias == "" {
alias = "media_items"
}
return fmt.Sprintf("(%s.status = 'matched' OR %s.type = 'audiobook' OR %s.type = 'ebook')", alias, alias, alias)
}
// UpsertEmbedding stores or updates an embedding for a media item.
func (r *Repo) UpsertEmbedding(ctx context.Context, itemID string, embedding []float32, model, canonicalText string) error {
padded, err := ensureCanonicalDimensions(embedding)
if err != nil {
return fmt.Errorf("upsert embedding for item %s: %w", itemID, err)
}
_, err = r.pool.Exec(ctx, `
INSERT INTO media_item_embeddings (media_item_id, embedding, model, canonical_text)
VALUES ($1, $2, $3, $4)
ON CONFLICT (media_item_id) DO UPDATE
SET embedding = EXCLUDED.embedding,
model = EXCLUDED.model,
canonical_text = EXCLUDED.canonical_text,
updated_at = NOW()
`, itemID, pgvector.NewVector(padded), model, canonicalText)
if err != nil {
return fmt.Errorf("upsert embedding for item %s: %w", itemID, err)
}
return nil
}
// GetEmbeddingLock retrieves the embedding lock metadata from server_settings.
func (r *Repo) GetEmbeddingLock(ctx context.Context) (*EmbeddingLock, error) {
var raw string
err := r.pool.QueryRow(ctx, `SELECT value FROM server_settings WHERE key = $1`, embeddingLockSettingKey).Scan(&raw)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("get embedding lock: %w", err)
}
lock, err := ParseEmbeddingLock(raw)
if err != nil {
return nil, err
}
return lock, nil
}
// SetEmbeddingLock stores the embedding lock metadata in server_settings.
func (r *Repo) SetEmbeddingLock(ctx context.Context, lock EmbeddingLock) error {
if lock.StorageDimensions == 0 {
lock.StorageDimensions = CanonicalEmbeddingDimensions
}
raw, err := lock.Marshal()
if err != nil {
return err
}
_, err = r.pool.Exec(ctx, `
INSERT INTO server_settings (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value
`, embeddingLockSettingKey, raw)
if err != nil {
return fmt.Errorf("set embedding lock: %w", err)
}
return nil
}
// GetEmbedding retrieves the embedding vector for a media item.
// Returns nil, nil when no embedding exists.
func (r *Repo) GetEmbedding(ctx context.Context, itemID string) ([]float32, error) {
var v pgvector.Vector
err := r.pool.QueryRow(ctx,
`SELECT embedding FROM media_item_embeddings WHERE media_item_id = $1`,
itemID,
).Scan(&v)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("get embedding for item %s: %w", itemID, err)
}
return v.Slice(), nil
}
// FindSimilar returns items ordered by cosine similarity to the given embedding,
// excluding the specified item IDs. When mediaType is non-empty, results are
// restricted to that media_items.type so cross-media-type results never appear
// (e.g. an audiobook in a "Similar to this movie" rail) once audiobook
// embeddings exist alongside movie/series ones.
func (r *Repo) FindSimilar(ctx context.Context, embedding []float32, excludeIDs []string, mediaType string, limit int) ([]ScoredItem, error) {
var items []ScoredItem
err := r.withHNSWCandidateScan(ctx, limit, func(tx pgx.Tx) error {
rows, err := tx.Query(ctx, `
SELECT e.media_item_id,
1 - (e.embedding::halfvec(3072) <=> $1::halfvec(3072)) AS similarity
FROM media_item_embeddings e
JOIN media_items mi ON mi.content_id = e.media_item_id
WHERE e.media_item_id != ALL($2)
AND ($4 = '' OR mi.type = $4)
ORDER BY e.embedding::halfvec(3072) <=> $1::halfvec(3072)
LIMIT $3
`, pgvector.NewVector(embedding), excludeIDs, limit, mediaType)
if err != nil {
return err
}
defer rows.Close()
items = make([]ScoredItem, 0, limit)
for rows.Next() {
var item ScoredItem
if err := rows.Scan(&item.MediaItemID, &item.Score); err != nil {
return fmt.Errorf("scanning similar item: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return fmt.Errorf("iterating similar items: %w", err)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("find similar items: %w", err)
}
return items, nil
}
// FindTasteProfileCandidates returns full-library discover candidates for a
// user's taste-profile embedding, optionally constrained to items sharing at
// least one selected genre and to the caller's access scope.
func (r *Repo) FindTasteProfileCandidates(
ctx context.Context,
embedding []float32,
excludeIDs []string,
genres []string,
limit int,
filter catalog.AccessFilter,
) ([]ScoredItem, map[string][]string, error) {
return r.findTasteProfileCandidates(ctx, embedding, excludeIDs, genres, limit, filter, "")
}
func (r *Repo) FindTasteProfileCandidatesByMediaType(
ctx context.Context,
embedding []float32,
excludeIDs []string,
genres []string,
limit int,
filter catalog.AccessFilter,
mediaType string,
) ([]ScoredItem, map[string][]string, error) {
return r.findTasteProfileCandidates(ctx, embedding, excludeIDs, genres, limit, filter, mediaType)
}
func (r *Repo) findTasteProfileCandidates(
ctx context.Context,
embedding []float32,
excludeIDs []string,
genres []string,
limit int,
filter catalog.AccessFilter,
mediaType string,
) ([]ScoredItem, map[string][]string, error) {
if embedding == nil || limit <= 0 {
return []ScoredItem{}, map[string][]string{}, nil
}
if excludeIDs == nil {
excludeIDs = []string{}
}
conditions := []string{
"(mi.status = 'matched' OR mi.type = 'audiobook')",
"e.media_item_id != ALL($2)",
}
args := []any{pgvector.NewVector(embedding), excludeIDs}
argIdx := 3
genreMatchCountSQL := "0"
annLimit := limit
if mediaType != "" {
conditions = append(conditions, fmt.Sprintf("mi.type = $%d", argIdx))
args = append(args, mediaType)
argIdx++
}
if len(genres) > 0 {
annLimit = limit * 5
if annLimit < limit {
annLimit = limit
}
if annLimit > 2000 {
annLimit = 2000
}
genreMatchCountSQL = fmt.Sprintf(`(
SELECT COUNT(DISTINCT matched_genre)
FROM unnest(COALESCE(mi.genres, '{}'::text[])) AS matched_genre
WHERE matched_genre = ANY($%d)
)`, argIdx)
conditions = append(conditions, fmt.Sprintf("mi.genres && $%d", argIdx))
args = append(args, genres)
argIdx++
}
if filter.AllowedLibraryIDs != nil {
if len(filter.AllowedLibraryIDs) == 0 {
return []ScoredItem{}, map[string][]string{}, nil
}
conditions = append(conditions, fmt.Sprintf(`
EXISTS (
SELECT 1
FROM media_item_libraries mil
WHERE mil.content_id = mi.content_id
AND mil.media_folder_id = ANY($%d)
)`, argIdx))
args = append(args, filter.AllowedLibraryIDs)
argIdx++
}
if len(filter.DisabledLibraryIDs) > 0 {
conditions = append(conditions, fmt.Sprintf(`
EXISTS (
SELECT 1
FROM media_item_libraries mil
WHERE mil.content_id = mi.content_id
AND mil.media_folder_id != ALL($%d)
)`, argIdx))
args = append(args, filter.DisabledLibraryIDs)
argIdx++
}
if filter.MaxContentRating != "" {
allowedRatings := access.AllowedRatingsUpTo(filter.MaxContentRating)
if len(allowedRatings) == 0 {
return []ScoredItem{}, map[string][]string{}, nil
}
placeholders := make([]string, len(allowedRatings))
for i, rating := range allowedRatings {
placeholders[i] = fmt.Sprintf("$%d", argIdx)
args = append(args, rating)
argIdx++
}
conditions = append(conditions, fmt.Sprintf(
"mi.content_rating IN (%s)",
strings.Join(placeholders, ", "),
))
}
query := fmt.Sprintf(`
WITH ann_candidates AS (
SELECT mi.content_id,
e.embedding::halfvec(3072) <=> $1::halfvec(3072) AS distance,
COALESCE(mi.genres, '{}'::text[]) AS genres
FROM media_item_embeddings e
JOIN media_items mi ON mi.content_id = e.media_item_id
WHERE %s
ORDER BY e.embedding::halfvec(3072) <=> $1::halfvec(3072)
LIMIT $%d
)
SELECT content_id,
1 - distance AS similarity,
genres,
%s AS genre_match_count
FROM ann_candidates mi
ORDER BY genre_match_count DESC, distance ASC, content_id
LIMIT $%d`,
strings.Join(conditions, " AND "),
argIdx,
genreMatchCountSQL,
argIdx+1,
)
args = append(args, annLimit, limit)
var items []ScoredItem
genreMap := make(map[string][]string, limit)
err := r.withHNSWCandidateScan(ctx, annLimit, func(tx pgx.Tx) error {
rows, err := tx.Query(ctx, query, args...)
if err != nil {
return err
}
defer rows.Close()
items = make([]ScoredItem, 0, limit)
for rows.Next() {
var item ScoredItem
var itemGenres []string
var genreMatchCount int
if err := rows.Scan(&item.MediaItemID, &item.Score, &itemGenres, &genreMatchCount); err != nil {
return fmt.Errorf("scan taste profile candidate: %w", err)
}
item.Reason = "taste_profile_match"
items = append(items, item)
genreMap[item.MediaItemID] = itemGenres
}
if err := rows.Err(); err != nil {
return fmt.Errorf("iterate taste profile candidates: %w", err)
}
return nil
})
if err != nil {
return nil, nil, fmt.Errorf("find taste profile candidates: %w", err)
}
return items, genreMap, nil
}
// ItemsNeedingEmbedding returns content IDs of media items that either have no
// embedding or whose embedding was generated with a different model.
// Books bypass the status='matched' gate because their scanner/plugin-derived
// metadata is authoritative as soon as the scan/enrichment completes.
func (r *Repo) ItemsNeedingEmbedding(ctx context.Context, currentModel string, limit int) ([]string, error) {
query := fmt.Sprintf(`
SELECT mi.content_id
FROM media_items mi
LEFT JOIN media_item_embeddings e ON e.media_item_id = mi.content_id
WHERE %s
AND (e.media_item_id IS NULL OR e.model != $1)
LIMIT $2
`, embeddingEligibilityWhereClause())
rows, err := r.pool.Query(ctx, query, currentModel, limit)
if err != nil {
return nil, fmt.Errorf("items needing embedding: %w", err)
}
defer rows.Close()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scanning content_id: %w", err)
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterating items needing embedding: %w", err)
}
return ids, nil
}
func (r *Repo) ListEmbeddingTextCandidates(ctx context.Context, afterID, currentModel string, limit int) ([]EmbeddingTextCandidate, error) {
query := fmt.Sprintf(`
-- Keep current_text in sync with embeddings.BuildEmbeddingText. This lets
-- the embedding job page over only missing, model-stale, or text-stale rows.
-- Book lines map to embeddings.mediaTypeLabel + the author/narrator
-- branch in BuildEmbeddingText; any divergence here forces book items
-- to re-embed every job run.
WITH text_candidates AS (
SELECT mi.content_id,
COALESCE(e.model, '') AS model,
COALESCE(e.canonical_text, '') AS canonical_text,
array_to_string(array_remove(ARRAY[
CASE
WHEN cardinality(COALESCE(mi.genres, ARRAY[]::text[])) > 0 AND COALESCE(mi.overview, '') <> ''
THEN array_to_string(mi.genres, ', ') || ' ' || CASE WHEN mi.type = 'series' THEN 'TV series' WHEN mi.type = 'audiobook' THEN 'audiobook' WHEN mi.type = 'ebook' THEN 'ebook' ELSE 'movie' END || ' about ' || substr(mi.overview, 1, 1000)
WHEN cardinality(COALESCE(mi.genres, ARRAY[]::text[])) > 0
THEN array_to_string(mi.genres, ', ') || ' ' || CASE WHEN mi.type = 'series' THEN 'TV series' WHEN mi.type = 'audiobook' THEN 'audiobook' WHEN mi.type = 'ebook' THEN 'ebook' ELSE 'movie' END
WHEN COALESCE(mi.overview, '') <> ''
THEN CASE WHEN mi.type = 'series' THEN 'TV series' WHEN mi.type = 'audiobook' THEN 'audiobook' WHEN mi.type = 'ebook' THEN 'ebook' ELSE 'movie' END || '. ' || substr(mi.overview, 1, 1000)
ELSE NULL
END,
CASE WHEN COALESCE(mi.year, 0) > 0
THEN COALESCE(mi.title, '') || ' (' || mi.year::text || ')'
ELSE COALESCE(mi.title, '')
END,
CASE WHEN COALESCE(mi.content_rating, '') <> '' THEN 'Rated ' || mi.content_rating ELSE NULL END,
CASE WHEN COALESCE(mi.tagline, '') <> '' THEN '"' || mi.tagline || '"' ELSE NULL END,
CASE WHEN mi.type NOT IN ('audiobook', 'ebook') AND actors.names <> '' THEN 'Cast: ' || actors.names ELSE NULL END,
CASE WHEN mi.type NOT IN ('audiobook', 'ebook') AND directors.names <> '' THEN 'Directed by ' || directors.names ELSE NULL END,
CASE
WHEN mi.type IN ('audiobook', 'ebook') AND authors.names <> '' THEN 'Written by ' || authors.names
WHEN mi.type NOT IN ('audiobook', 'ebook') AND writers.names <> '' THEN 'Written by ' || writers.names
ELSE NULL
END,
CASE WHEN mi.type = 'audiobook' AND narrators.names <> '' THEN 'Narrated by ' || narrators.names ELSE NULL END,
CASE WHEN cardinality(COALESCE(mi.keywords, ARRAY[]::text[])) > 0 THEN 'Keywords: ' || array_to_string((mi.keywords)[1:5], ', ') ELSE NULL END,
CASE WHEN COALESCE(mi.original_language, '') <> '' THEN 'Original language: ' || mi.original_language ELSE NULL END,
CASE WHEN cardinality(COALESCE(mi.studios, ARRAY[]::text[])) > 0 THEN 'Studios: ' || array_to_string(mi.studios, ', ') ELSE NULL END,
CASE WHEN cardinality(COALESCE(mi.networks, ARRAY[]::text[])) > 0 THEN 'Network: ' || array_to_string(mi.networks, ', ') ELSE NULL END,
CASE WHEN cardinality(COALESCE(mi.countries, ARRAY[]::text[])) > 0 THEN 'Country: ' || array_to_string((mi.countries)[1:2], ', ') ELSE NULL END
]::text[], NULL), '. ') AS current_text
FROM media_items mi
LEFT JOIN media_item_embeddings e ON e.media_item_id = mi.content_id
LEFT JOIN LATERAL (
SELECT COALESCE(string_agg(
CASE WHEN ranked.character <> '' THEN ranked.name || ' as ' || ranked.character ELSE ranked.name END,
', ' ORDER BY ranked.sort_order
), '') AS names
FROM (
SELECT COALESCE(p.name, '') AS name,
COALESCE(ip.character, '') AS character,
ip.sort_order,
ip.person_id
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
AND ip.kind = 1
ORDER BY ip.sort_order, p.name, COALESCE(ip.character, ''), ip.person_id
LIMIT 5
) ranked
) actors ON TRUE
LEFT JOIN LATERAL (
SELECT COALESCE(string_agg(COALESCE(p.name, ''), ', ' ORDER BY ip.sort_order, p.name, ip.person_id), '') AS names
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
AND ip.kind = 2
) directors ON TRUE
LEFT JOIN LATERAL (
SELECT COALESCE(string_agg(COALESCE(p.name, ''), ', ' ORDER BY ip.sort_order, p.name, ip.person_id), '') AS names
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
AND ip.kind = 3
) writers ON TRUE
LEFT JOIN LATERAL (
SELECT COALESCE(string_agg(COALESCE(p.name, ''), ', ' ORDER BY ip.sort_order, p.name, ip.person_id), '') AS names
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
AND ip.kind = 7
) authors ON TRUE
LEFT JOIN LATERAL (
SELECT COALESCE(string_agg(COALESCE(p.name, ''), ', ' ORDER BY ip.sort_order, p.name, ip.person_id), '') AS names
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
AND ip.kind = 8
) narrators ON TRUE
WHERE %s
AND ($1 = '' OR mi.content_id > $1)
)
SELECT mi.content_id,
mi.model,
mi.canonical_text
FROM text_candidates mi
WHERE mi.model = ''
OR mi.model != $2
OR mi.canonical_text IS DISTINCT FROM mi.current_text
ORDER BY mi.content_id
LIMIT $3
`, embeddingEligibilityWhereClause())
rows, err := r.pool.Query(ctx, query, afterID, currentModel, limit)
if err != nil {
return nil, fmt.Errorf("embedding text candidates: %w", err)
}
defer rows.Close()
var candidates []EmbeddingTextCandidate
for rows.Next() {
var candidate EmbeddingTextCandidate
if err := rows.Scan(&candidate.MediaItemID, &candidate.Model, &candidate.CanonicalText); err != nil {
return nil, fmt.Errorf("scanning embedding text candidate: %w", err)
}
candidates = append(candidates, candidate)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterating embedding text candidates: %w", err)
}
return candidates, nil
}
// EmbeddingCount returns the total number of stored embeddings.
func (r *Repo) EmbeddingCount(ctx context.Context) (int, error) {
var count int
err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM media_item_embeddings`).Scan(&count)
if err != nil {
return 0, fmt.Errorf("embedding count: %w", err)
}
return count, nil
}
// TotalMediaItemCount returns the number of media items eligible for embedding.
// Mirrors the status/type gate in ItemsNeedingEmbedding so progress reporting
// (used by the admin worker UI) lines up with what the job actually does.
func (r *Repo) TotalMediaItemCount(ctx context.Context) (int, error) {
var count int
query := fmt.Sprintf(`SELECT COUNT(*) FROM media_items mi WHERE %s`, embeddingEligibilityWhereClause())
err := r.pool.QueryRow(ctx, query).Scan(&count)
if err != nil {
return 0, fmt.Errorf("total media item count: %w", err)
}
return count, nil
}
// TasteProfileCount returns the total number of stored taste profiles.
func (r *Repo) TasteProfileCount(ctx context.Context) (int, error) {
var count int
err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM user_taste_profiles`).Scan(&count)
if err != nil {
return 0, fmt.Errorf("taste profile count: %w", err)
}
return count, nil
}
// CacheEntryCount returns the total number of non-expired recommendation cache entries.
func (r *Repo) CacheEntryCount(ctx context.Context) (int, error) {
var count int
err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM recommendation_cache WHERE expires_at > NOW()`).Scan(&count)
if err != nil {
return 0, fmt.Errorf("cache entry count: %w", err)
}
return count, nil
}
// UpsertTasteProfile stores or updates a user's precomputed taste profile.
func (r *Repo) UpsertTasteProfile(ctx context.Context, userID int, profileID string, embedding []float32, signalCounts map[string]int, maxContentRating string) error {
countsJSON, err := json.Marshal(signalCounts)
if err != nil {
return fmt.Errorf("marshaling signal counts: %w", err)
}
_, err = r.pool.Exec(ctx, `
INSERT INTO user_taste_profiles
(user_id, profile_id, embedding, signal_counts, max_content_rating, updated_at)
VALUES ($1, $2, $3, $4, $5, NOW())
ON CONFLICT (user_id, profile_id) DO UPDATE
SET embedding = EXCLUDED.embedding,
signal_counts = EXCLUDED.signal_counts,
max_content_rating = EXCLUDED.max_content_rating,
updated_at = NOW()
`, userID, profileID, pgvector.NewVector(embedding), countsJSON, maxContentRating)
if err != nil {
return fmt.Errorf("upsert taste profile for user %d profile %s: %w", userID, profileID, err)
}
return nil
}
// TasteProfileMeta holds the non-vector metadata for a taste profile.
type TasteProfileMeta struct {
SignalCounts map[string]int
MaxContentRating string
UpdatedAt string
}
// GetTasteProfileMeta retrieves the signal_counts and updated_at for a user's
// taste profile. Returns nil, nil when no profile exists.
func (r *Repo) GetTasteProfileMeta(ctx context.Context, userID int, profileID string) (*TasteProfileMeta, error) {
var countsJSON []byte
var updatedAt time.Time
var maxContentRating string
err := r.pool.QueryRow(ctx,
`SELECT signal_counts, COALESCE(max_content_rating, ''), updated_at FROM user_taste_profiles WHERE user_id = $1 AND profile_id = $2`,
userID, profileID,
).Scan(&countsJSON, &maxContentRating, &updatedAt)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("get taste profile meta for user %d profile %s: %w", userID, profileID, err)
}
var counts map[string]int
if err := json.Unmarshal(countsJSON, &counts); err != nil {
return nil, fmt.Errorf("unmarshal signal_counts: %w", err)
}
return &TasteProfileMeta{
SignalCounts: counts,
MaxContentRating: maxContentRating,
UpdatedAt: updatedAt.Format(time.RFC3339),
}, nil
}
// GetTasteProfile retrieves the embedding for a user's taste profile.
// Returns nil, nil when no profile exists.
func (r *Repo) GetTasteProfile(ctx context.Context, userID int, profileID string) ([]float32, error) {
var v pgvector.Vector
err := r.pool.QueryRow(ctx,
`SELECT embedding FROM user_taste_profiles WHERE user_id = $1 AND profile_id = $2`,
userID, profileID,
).Scan(&v)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("get taste profile for user %d profile %s: %w", userID, profileID, err)
}
return v.Slice(), nil
}
// FindSimilarUsers returns users with taste profiles similar to the given user's
// profile, filtered by max_content_rating compatibility.
func (r *Repo) FindSimilarUsers(ctx context.Context, userID int, profileID string, maxContentRating string, limit int) ([]UserSimilarity, error) {
embedding, err := r.GetTasteProfile(ctx, userID, profileID)
if err != nil {
return nil, fmt.Errorf("getting taste profile for similarity search: %w", err)
}
if embedding == nil {
return nil, nil
}
allowedPeerRatings := compatiblePeerContentRatings(maxContentRating)
var users []UserSimilarity
err = r.withHNSWCandidateScan(ctx, limit, func(tx pgx.Tx) error {
rows, err := tx.Query(ctx, `
SELECT p.user_id,
p.profile_id,
1 - (p.embedding::halfvec(3072) <=> $1::halfvec(3072)) AS score
FROM user_taste_profiles p
WHERE p.user_id != $2
AND ($3 OR COALESCE(p.max_content_rating, '') = '' OR p.max_content_rating = ANY($4::text[]))
ORDER BY p.embedding::halfvec(3072) <=> $1::halfvec(3072)
LIMIT $5
`, pgvector.NewVector(embedding), userID, maxContentRating == "", allowedPeerRatings, limit)
if err != nil {
return err
}
defer rows.Close()
users = make([]UserSimilarity, 0, limit)
for rows.Next() {
var u UserSimilarity
if err := rows.Scan(&u.UserID, &u.ProfileID, &u.Score); err != nil {
return fmt.Errorf("scanning similar user: %w", err)
}
users = append(users, u)
}
if err := rows.Err(); err != nil {
return fmt.Errorf("iterating similar users: %w", err)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("find similar users: %w", err)
}
return users, nil
}
func compatiblePeerContentRatings(maxContentRating string) []string {
allowed := access.AllowedRatingsUpTo(maxContentRating)
if len(allowed) == 0 {
return []string{}
}
sort.Strings(allowed)
return allowed
}
// UpsertRecommendationCache stores or refreshes a precomputed recommendation
// list for a user.
func (r *Repo) UpsertRecommendationCache(ctx context.Context, userID int, profileID, recType, sourceItemID string, items []ScoredItem, expiresAt string) error {
itemsJSON, err := json.Marshal(items)
if err != nil {
return fmt.Errorf("marshaling cached items: %w", err)
}
_, err = r.pool.Exec(ctx, `
INSERT INTO recommendation_cache
(user_id, profile_id, rec_type, source_item_id, items, expires_at, created_at)
VALUES ($1, $2, $3, $4, $5, $6::timestamptz, NOW())
ON CONFLICT (user_id, profile_id, rec_type, source_item_id) DO UPDATE
SET items = EXCLUDED.items,
expires_at = EXCLUDED.expires_at,
created_at = NOW()
`, userID, profileID, recType, sourceItemID, itemsJSON, expiresAt)
if err != nil {
return fmt.Errorf("upsert recommendation cache: %w", err)
}
return nil
}
// GetRecommendationCache retrieves cached recommendation results that have not
// yet expired. Returns nil, nil on cache miss or expiry.
func (r *Repo) GetRecommendationCache(ctx context.Context, userID int, profileID, recType, sourceItemID string) ([]ScoredItem, error) {
var itemsJSON []byte
err := r.pool.QueryRow(ctx, `
SELECT items
FROM recommendation_cache
WHERE user_id = $1
AND profile_id = $2
AND rec_type = $3
AND source_item_id = $4
AND expires_at > NOW()
`, userID, profileID, recType, sourceItemID).Scan(&itemsJSON)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("get recommendation cache: %w", err)
}
var items []ScoredItem
if err := json.Unmarshal(itemsJSON, &items); err != nil {
return nil, fmt.Errorf("unmarshaling cached items: %w", err)
}
return items, nil
}
// ListCachedGenreSamplers returns all non-expired global genre sampler cache entries
// as a map of genre name → scored items.
func (r *Repo) ListCachedGenreSamplers(ctx context.Context) (map[string][]ScoredItem, error) {
rows, err := r.pool.Query(ctx, `
SELECT rec_type, items
FROM recommendation_cache
WHERE user_id = $1
AND profile_id = $2
AND rec_type LIKE $3
AND expires_at > NOW()`,
GlobalCacheUserID, GlobalCacheProfileID, RecTypeGenreSamplerPrefix+"%")
if err != nil {
return nil, fmt.Errorf("list cached genre samplers: %w", err)
}
defer rows.Close()
result := make(map[string][]ScoredItem)
for rows.Next() {
var recType string
var itemsJSON []byte
if err := rows.Scan(&recType, &itemsJSON); err != nil {
return nil, fmt.Errorf("scan genre sampler cache: %w", err)
}
genre := strings.TrimPrefix(recType, RecTypeGenreSamplerPrefix)
var items []ScoredItem
if err := json.Unmarshal(itemsJSON, &items); err != nil {
continue
}
result[genre] = items
}
return result, rows.Err()
}
// CleanExpiredCache removes all expired recommendation cache entries and
// returns the number of rows deleted.
func (r *Repo) CleanExpiredCache(ctx context.Context) (int64, error) {
tag, err := r.pool.Exec(ctx, `DELETE FROM recommendation_cache WHERE expires_at < NOW()`)
if err != nil {
return 0, fmt.Errorf("clean expired cache: %w", err)
}
return tag.RowsAffected(), nil
}
// --- Taste Cluster Operations ---
// UpsertTasteClusters replaces all clusters for a user/profile.
func (r *Repo) UpsertTasteClusters(ctx context.Context, userID int, profileID string, clusters []TasteCluster) error {
tx, err := r.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("begin tx for taste clusters: %w", err)
}
defer tx.Rollback(ctx)
_, err = tx.Exec(ctx,
`DELETE FROM user_taste_clusters WHERE user_id = $1 AND profile_id = $2`,
userID, profileID)
if err != nil {
return fmt.Errorf("delete old clusters: %w", err)
}
for _, c := range clusters {
genresJSON, err := json.Marshal(c.DominantGenres)
if err != nil {
return fmt.Errorf("marshal genres: %w", err)
}
_, err = tx.Exec(ctx, `
INSERT INTO user_taste_clusters
(user_id, profile_id, cluster_idx, embedding, dominant_genres, label, member_count, total_weight, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())`,
userID, profileID, c.ClusterIdx,
pgvector.NewVector(c.Embedding), genresJSON, c.Label,
c.MemberCount, c.TotalWeight)
if err != nil {
return fmt.Errorf("insert cluster %d: %w", c.ClusterIdx, err)
}
}
return tx.Commit(ctx)
}
// GetTasteClusters retrieves all clusters for a user/profile.
func (r *Repo) GetTasteClusters(ctx context.Context, userID int, profileID string) ([]TasteCluster, error) {
rows, err := r.pool.Query(ctx, `
SELECT cluster_idx, embedding, dominant_genres, label, member_count, total_weight, updated_at
FROM user_taste_clusters
WHERE user_id = $1 AND profile_id = $2
ORDER BY cluster_idx`,
userID, profileID)
if err != nil {
return nil, fmt.Errorf("get taste clusters: %w", err)
}
defer rows.Close()
var clusters []TasteCluster
for rows.Next() {
var c TasteCluster
var v pgvector.Vector
var genresJSON []byte
if err := rows.Scan(&c.ClusterIdx, &v, &genresJSON, &c.Label, &c.MemberCount, &c.TotalWeight, &c.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan cluster: %w", err)
}
c.UserID = userID
c.ProfileID = profileID
c.Embedding = v.Slice()
if err := json.Unmarshal(genresJSON, &c.DominantGenres); err != nil {
return nil, fmt.Errorf("unmarshal cluster genres: %w", err)
}
clusters = append(clusters, c)
}
return clusters, rows.Err()
}
// --- Co-Watch Operations ---
// UpsertCowatchPairs bulk-upserts co-watch pairs. Operates in a single transaction.
func (r *Repo) UpsertCowatchPairs(ctx context.Context, pairs []CowatchPair) error {
if len(pairs) == 0 {
return nil
}
tx, err := r.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("begin tx for cowatch: %w", err)
}
defer tx.Rollback(ctx)
for _, p := range pairs {
_, err := tx.Exec(ctx, `
INSERT INTO item_cowatch (item_id, similar_item_id, jaccard_score, cowatch_count, updated_at)
VALUES ($1, $2, $3, $4, NOW())
ON CONFLICT (item_id, similar_item_id) DO UPDATE
SET jaccard_score = EXCLUDED.jaccard_score,
cowatch_count = EXCLUDED.cowatch_count,
updated_at = NOW()`,
p.ItemID, p.SimilarItemID, p.JaccardScore, p.CowatchCount)
if err != nil {
return fmt.Errorf("upsert cowatch pair: %w", err)
}
}
return tx.Commit(ctx)
}
// GetCowatchNeighbors returns the top co-watch neighbors for an item.
func (r *Repo) GetCowatchNeighbors(ctx context.Context, itemID string, limit int) ([]CowatchPair, error) {
rows, err := r.pool.Query(ctx, `
SELECT item_id, similar_item_id, jaccard_score, cowatch_count
FROM item_cowatch
WHERE item_id = $1
ORDER BY jaccard_score DESC
LIMIT $2`,
itemID, limit)
if err != nil {
return nil, fmt.Errorf("get cowatch neighbors: %w", err)
}
defer rows.Close()
var pairs []CowatchPair
for rows.Next() {
var p CowatchPair
if err := rows.Scan(&p.ItemID, &p.SimilarItemID, &p.JaccardScore, &p.CowatchCount); err != nil {
return nil, fmt.Errorf("scan cowatch pair: %w", err)
}
pairs = append(pairs, p)
}
return pairs, rows.Err()
}
// CowatchPairCount returns the total number of co-watch pairs stored.
func (r *Repo) CowatchPairCount(ctx context.Context) (int, error) {
var count int
err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM item_cowatch`).Scan(&count)
if err != nil {
return 0, fmt.Errorf("cowatch pair count: %w", err)
}
return count, nil
}
// --- Staleness Operations ---
// MarkProfileStale sets stale_at = NOW() on a user's taste profile.
func (r *Repo) MarkProfileStale(ctx context.Context, userID int, profileID string) error {
_, err := r.pool.Exec(ctx,
`UPDATE user_taste_profiles SET stale_at = NOW() WHERE user_id = $1 AND profile_id = $2`,
userID, profileID)
if err != nil {
return fmt.Errorf("mark profile stale: %w", err)
}
return nil
}
// StaleProfile represents a taste profile that needs refreshing.
type StaleProfile struct {
UserID int
ProfileID string
}
// GetStaleProfiles returns profiles where stale_at > updated_at.
func (r *Repo) GetStaleProfiles(ctx context.Context, limit int) ([]StaleProfile, error) {
rows, err := r.pool.Query(ctx, `
SELECT user_id, profile_id
FROM user_taste_profiles
WHERE stale_at IS NOT NULL AND stale_at > updated_at
LIMIT $1`, limit)
if err != nil {
return nil, fmt.Errorf("get stale profiles: %w", err)
}
defer rows.Close()
var profiles []StaleProfile
for rows.Next() {
var p StaleProfile
if err := rows.Scan(&p.UserID, &p.ProfileID); err != nil {
return nil, fmt.Errorf("scan stale profile: %w", err)
}
profiles = append(profiles, p)
}
return profiles, rows.Err()
}
// ClearStaleAt resets stale_at to NULL after refreshing a profile.
func (r *Repo) ClearStaleAt(ctx context.Context, userID int, profileID string) error {
_, err := r.pool.Exec(ctx,
`UPDATE user_taste_profiles SET stale_at = NULL WHERE user_id = $1 AND profile_id = $2`,
userID, profileID)
if err != nil {
return fmt.Errorf("clear stale_at: %w", err)
}
return nil
}
// --- Watch Signal Queries (cross-user aggregation) ---
// WatchProgressRow holds raw watch progress data for signal computation.
type WatchProgressRow struct {
MediaItemID string
PositionSeconds float64
DurationSeconds float64
Completed bool
UpdatedAt time.Time
}
// ResolveCanonicalContentRefs maps mixed movie/series/season/episode IDs to
// the canonical movie/series entity used for taste-profile learning.
func (r *Repo) ResolveCanonicalContentRefs(ctx context.Context, contentIDs []string) (map[string]canonicalContentRef, error) {
result := make(map[string]canonicalContentRef, len(contentIDs))
if len(contentIDs) == 0 {
return result, nil
}
rows, err := r.pool.Query(ctx, `
SELECT content_id, kind, canonical_id, series_id, season_number
FROM (
SELECT mi.content_id,
mi.type AS kind,
mi.content_id AS canonical_id,
NULL::text AS series_id,
NULL::int AS season_number
FROM media_items mi
WHERE mi.content_id = ANY($1)
UNION ALL
SELECT s.content_id,
'season' AS kind,
s.series_id AS canonical_id,
s.series_id AS series_id,
s.season_number
FROM seasons s
WHERE s.content_id = ANY($1)
UNION ALL
SELECT e.content_id,
'episode' AS kind,
e.series_id AS canonical_id,
e.series_id AS series_id,
e.season_number
FROM episodes e
WHERE e.content_id = ANY($1)
) refs
`, contentIDs)
if err != nil {
return nil, fmt.Errorf("resolve canonical content refs: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
contentID string
kind string
canonicalID string
seriesID *string
seasonNumber *int
)
if err := rows.Scan(&contentID, &kind, &canonicalID, &seriesID, &seasonNumber); err != nil {
return nil, fmt.Errorf("scan canonical content ref: %w", err)
}
ref := canonicalContentRef{
Kind: canonicalContentKind(kind),
CanonicalID: canonicalID,
}
if seriesID != nil {
ref.SeriesID = *seriesID
}
if seasonNumber != nil {
ref.SeasonNumber = *seasonNumber
ref.HasSeason = true
}
result[contentID] = ref
}
return result, rows.Err()
}
// GetWatchProgressForUser returns all watch progress entries for a user/profile.
func (r *Repo) GetWatchProgressForUser(ctx context.Context, userID int, profileID string) ([]WatchProgressRow, error) {
rows, err := r.pool.Query(ctx, `
SELECT media_item_id, position_seconds, duration_seconds, completed, updated_at
FROM user_watch_progress
WHERE user_id = $1 AND profile_id = $2
AND NOT EXISTS (
SELECT 1
FROM user_history_hidden_items hhi
WHERE hhi.user_id = user_watch_progress.user_id
AND hhi.profile_id = user_watch_progress.profile_id
AND hhi.media_item_id = user_watch_progress.media_item_id
AND user_watch_progress.updated_at <= hhi.hidden_before
)`,
userID, profileID)
if err != nil {
return nil, fmt.Errorf("get watch progress: %w", err)
}
defer rows.Close()
var result []WatchProgressRow
for rows.Next() {
var wp WatchProgressRow
if err := rows.Scan(&wp.MediaItemID, &wp.PositionSeconds, &wp.DurationSeconds, &wp.Completed, &wp.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan watch progress: %w", err)
}
result = append(result, wp)
}
return result, rows.Err()
}
// ebookReaderProgressForUserQuery maps ebook reader progress onto the
// WatchProgressRow shape (progress as position with duration 1). Rows hidden
// via user_history_hidden_items are excluded with the same semantics as
// GetWatchProgressForUser: hidden when updated_at <= hidden_before, visible
// again once new reading activity lands after hidden_before.
var ebookReaderProgressForUserQuery = fmt.Sprintf(`
SELECT content_id,
progress::double precision AS position_seconds,
1::double precision AS duration_seconds,
progress >= %s AS completed,
updated_at
FROM ebook_reader_progress
WHERE user_id = $1 AND profile_id = $2
AND NOT EXISTS (
SELECT 1
FROM user_history_hidden_items hhi
WHERE hhi.user_id = ebook_reader_progress.user_id
AND hhi.profile_id = ebook_reader_progress.profile_id
AND hhi.media_item_id = ebook_reader_progress.content_id
AND ebook_reader_progress.updated_at <= hhi.hidden_before
)`, catalog.EbookFinishedProgressThresholdSQL)
func (r *Repo) GetEbookReaderProgressForUser(ctx context.Context, userID int, profileID string) ([]WatchProgressRow, error) {
rows, err := r.pool.Query(ctx, ebookReaderProgressForUserQuery, userID, profileID)
if err != nil {
return nil, fmt.Errorf("get ebook reader progress: %w", err)
}
defer rows.Close()
var result []WatchProgressRow
for rows.Next() {
var wp WatchProgressRow
if err := rows.Scan(&wp.MediaItemID, &wp.PositionSeconds, &wp.DurationSeconds, &wp.Completed, &wp.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan ebook reader progress: %w", err)
}
result = append(result, wp)
}
return result, rows.Err()
}
// RewatchCount holds the number of completed watches for an item.
type RewatchCount struct {
MediaItemID string
Count int
LastWatchedAt time.Time
}
// GetRewatchCounts returns items watched to completion 2+ times by a user/profile.
func (r *Repo) GetRewatchCounts(ctx context.Context, userID int, profileID string) ([]RewatchCount, error) {
rows, err := r.pool.Query(ctx, `
SELECT media_item_id, COUNT(*) AS cnt, MAX(watched_at) AS last_watched_at
FROM user_watch_history
WHERE user_id = $1 AND profile_id = $2 AND completed = true
AND NOT EXISTS (
SELECT 1
FROM user_history_hidden_items hhi
WHERE hhi.user_id = user_watch_history.user_id
AND hhi.profile_id = user_watch_history.profile_id
AND hhi.media_item_id = user_watch_history.media_item_id
AND user_watch_history.watched_at <= hhi.hidden_before
)
GROUP BY media_item_id
HAVING COUNT(*) >= 2`,
userID, profileID)
if err != nil {
return nil, fmt.Errorf("get rewatch counts: %w", err)
}
defer rows.Close()
var result []RewatchCount
for rows.Next() {
var rc RewatchCount
if err := rows.Scan(&rc.MediaItemID, &rc.Count, &rc.LastWatchedAt); err != nil {
return nil, fmt.Errorf("scan rewatch count: %w", err)
}
result = append(result, rc)
}
return result, rows.Err()
}
// itemWatchersQuery feeds co-watch matrix computation. watched_activity rolls
// episodes up to their parent series, so a single watcher can produce many
// rows for the same item; the GROUP BY collapses them to one row per
// (watcher, item) before ranking and aggregation so a lone binge-watcher
// cannot satisfy the minimum-watchers threshold ($2) by itself and the
// per-user recency cap ($1) counts distinct items rather than raw rows.
//
// Watcher identity is (user_id, profile_id) for the Jaccard math — profiles of
// one account legitimately have distinct tastes — but the minimum-watchers
// floor ($2) is a sparsity/privacy threshold and must count distinct login
// ACCOUNTS: one household account with N profiles must not satisfy it alone,
// so the HAVING clause counts DISTINCT user_id rather than watcher rows.
var itemWatchersQuery = fmt.Sprintf(`
WITH %s,
user_watches AS (
SELECT user_id,
watcher_id,
item_id AS media_item_id,
ROW_NUMBER() OVER (PARTITION BY user_id, profile_id ORDER BY MAX(updated_at) DESC) AS rn
FROM watched_activity
GROUP BY user_id, profile_id, watcher_id, item_id
)
SELECT media_item_id, ARRAY_AGG(watcher_id) AS watchers
FROM user_watches
WHERE rn <= $1
GROUP BY media_item_id
HAVING COUNT(DISTINCT user_id) >= $2`, watchedActivityCTE)
// ItemWatchers maps item_id to profile identities that watched >= 50% progress.
// Used for co-watch matrix computation.
func (r *Repo) GetItemWatchers(ctx context.Context, minWatchers int, maxPerUser int) (map[string][]string, error) {
rows, err := r.pool.Query(ctx, itemWatchersQuery, maxPerUser, minWatchers)
if err != nil {
return nil, fmt.Errorf("get item watchers: %w", err)
}
defer rows.Close()
result := make(map[string][]string)
for rows.Next() {
var itemID string
var watchers []string
if err := rows.Scan(&itemID, &watchers); err != nil {
return nil, fmt.Errorf("scan item watchers: %w", err)
}
result[itemID] = watchers
}
return result, rows.Err()
}
// --- Cold Start Queries ---
// GetPopularItems returns the most-played media items over the given number of days.
// Episodes are resolved to their parent series.
func (r *Repo) GetPopularItems(ctx context.Context, days, limit int) ([]ScoredItem, error) {
query := fmt.Sprintf(`
WITH %s,
watched_items AS (
SELECT item_id, watcher_id
FROM watched_activity
WHERE updated_at > NOW() - ($1 || ' days')::interval
)
SELECT wi.item_id, COUNT(DISTINCT wi.watcher_id) AS watch_count
FROM watched_items wi
JOIN media_items mi ON mi.content_id = wi.item_id
GROUP BY wi.item_id
ORDER BY watch_count DESC
LIMIT $2`, watchedActivityCTE)
rows, err := r.pool.Query(ctx, query, fmt.Sprintf("%d", days), limit)
if err != nil {
return nil, fmt.Errorf("get popular items: %w", err)
}
defer rows.Close()
var items []ScoredItem
for rows.Next() {
var item ScoredItem
var watchCount int
if err := rows.Scan(&item.MediaItemID, &watchCount); err != nil {
return nil, fmt.Errorf("scan popular item: %w", err)
}
item.Score = float64(watchCount)
item.Reason = "popular"
items = append(items, item)
}
return items, rows.Err()
}
// GetRecentlyAddedItems returns items added within the given number of days.
// Audiobooks and ebooks bypass the matched-status gate because their
// scan-derived metadata is authoritative before any external-provider match
// exists.
func (r *Repo) GetRecentlyAddedItems(ctx context.Context, days, limit int) ([]ScoredItem, error) {
query := fmt.Sprintf(`
SELECT mi.content_id, mi.created_at
FROM media_items mi
WHERE %s
AND mi.created_at > NOW() - ($1 || ' days')::interval
ORDER BY mi.created_at DESC
LIMIT $2`, recommendationItemEligibilityWhereClause("mi"))
rows, err := r.pool.Query(ctx, query, fmt.Sprintf("%d", days), limit)
if err != nil {
return nil, fmt.Errorf("get recently added: %w", err)
}
defer rows.Close()
var items []ScoredItem
for rows.Next() {
var item ScoredItem
var createdAt time.Time
if err := rows.Scan(&item.MediaItemID, &createdAt); err != nil {
return nil, fmt.Errorf("scan recently added: %w", err)
}
item.Score = float64(createdAt.Unix())
item.Reason = "recently_added"
items = append(items, item)
}
return items, rows.Err()
}
// GetTopRatedItems returns items with highest average rating (min ratingCount ratings).
func (r *Repo) GetTopRatedItems(ctx context.Context, minRatings, limit int) ([]ScoredItem, error) {
rows, err := r.pool.Query(ctx, `
SELECT media_item_id, AVG(rating) AS avg_rating
FROM user_ratings
GROUP BY media_item_id
HAVING COUNT(*) >= $1
ORDER BY avg_rating DESC
LIMIT $2`,
minRatings, limit)
if err != nil {
return nil, fmt.Errorf("get top rated: %w", err)
}
defer rows.Close()
var items []ScoredItem
for rows.Next() {
var item ScoredItem
if err := rows.Scan(&item.MediaItemID, &item.Score); err != nil {
return nil, fmt.Errorf("scan top rated: %w", err)
}
item.Reason = "top_rated"
items = append(items, item)
}
return items, rows.Err()
}
// GetTasteSeedCandidates returns movie/series/audiobook content IDs ordered for the
// taste-seeding picker: server engagement first (most-watched in the last
// 180 days), then rating reliability and rating score, then recency. This keeps
// fresh servers from front-loading single-vote TMDB 10.0 obscurities while
// established servers prioritize what users actually watch. Episodes are resolved
// to their parent series. Items without a poster are excluded.
func (r *Repo) GetTasteSeedCandidates(ctx context.Context, limit, offset int) ([]string, error) {
rows, err := r.pool.Query(ctx, tasteSeedCandidateQuery, limit, offset)
if err != nil {
return nil, fmt.Errorf("get taste seed candidates: %w", err)
}
defer rows.Close()
ids := make([]string, 0, limit)
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan taste seed candidate: %w", err)
}
ids = append(ids, id)
}
return ids, rows.Err()
}
// GetTopGenres returns the most popular genres by watch count.
// Episodes are resolved to their parent series for genre lookup.
func (r *Repo) GetTopGenres(ctx context.Context, limit int) ([]string, error) {
query := fmt.Sprintf(`
WITH %s,
watched_items AS (
SELECT item_id, watcher_id
FROM watched_activity
)
SELECT g.genre, COUNT(DISTINCT wi.watcher_id) AS watchers
FROM watched_items wi
JOIN media_items mi ON mi.content_id = wi.item_id
CROSS JOIN LATERAL UNNEST(mi.genres) AS g(genre)
GROUP BY g.genre
ORDER BY watchers DESC
LIMIT $1`, watchedActivityCTE)
rows, err := r.pool.Query(ctx, query, limit)
if err != nil {
return nil, fmt.Errorf("get top genres: %w", err)
}
defer rows.Close()
var genres []string
for rows.Next() {
var genre string
var count int
if err := rows.Scan(&genre, &count); err != nil {
return nil, fmt.Errorf("scan top genre: %w", err)
}
genres = append(genres, genre)
}
return genres, rows.Err()
}
// GetGenreSamplerItems returns the most-played media items in a specific genre.
func (r *Repo) GetGenreSamplerItems(ctx context.Context, genre string, limit int) ([]ScoredItem, error) {
query := fmt.Sprintf(`
WITH %s,
watched_items AS (
SELECT item_id, watcher_id
FROM watched_activity
)
SELECT wi.item_id, COUNT(DISTINCT wi.watcher_id) AS watch_count
FROM watched_items wi
JOIN media_items mi ON mi.content_id = wi.item_id
WHERE $1 = ANY(mi.genres)
GROUP BY wi.item_id
ORDER BY watch_count DESC
LIMIT $2`, watchedActivityCTE)
rows, err := r.pool.Query(ctx, query, genre, limit)
if err != nil {
return nil, fmt.Errorf("get genre sampler items: %w", err)
}
defer rows.Close()
var items []ScoredItem
for rows.Next() {
var item ScoredItem
var watchCount int
if err := rows.Scan(&item.MediaItemID, &watchCount); err != nil {
return nil, fmt.Errorf("scan genre sampler item: %w", err)
}
item.Score = float64(watchCount)
item.Reason = "genre_sampler"
items = append(items, item)
}
return items, rows.Err()
}
// GetBatchEmbeddings retrieves embeddings for multiple item IDs.
func (r *Repo) GetBatchEmbeddings(ctx context.Context, itemIDs []string) (map[string][]float32, error) {
if len(itemIDs) == 0 {
return nil, nil
}
rows, err := r.pool.Query(ctx, `
SELECT media_item_id, embedding
FROM media_item_embeddings
WHERE media_item_id = ANY($1)`,
itemIDs)
if err != nil {
return nil, fmt.Errorf("get batch embeddings: %w", err)
}
defer rows.Close()
result := make(map[string][]float32)
for rows.Next() {
var id string
var v pgvector.Vector
if err := rows.Scan(&id, &v); err != nil {
return nil, fmt.Errorf("scan batch embedding: %w", err)
}
result[id] = v.Slice()
}
return result, rows.Err()
}
// GetItemGenres returns the full genre array for each item ID.
func (r *Repo) GetItemGenres(ctx context.Context, itemIDs []string) (map[string][]string, error) {
if len(itemIDs) == 0 {
return nil, nil
}
rows, err := r.pool.Query(ctx, `
SELECT content_id, genres
FROM media_items
WHERE content_id = ANY($1)
AND array_length(genres, 1) > 0`,
itemIDs)
if err != nil {
return nil, fmt.Errorf("get item genres: %w", err)
}
defer rows.Close()
result := make(map[string][]string)
for rows.Next() {
var id string
var genres []string
if err := rows.Scan(&id, &genres); err != nil {
return nil, fmt.Errorf("scan item genre: %w", err)
}
result[id] = genres
}
return result, rows.Err()
}
// FilterAccessibleItemIDs returns the subset of item IDs allowed by the given
// access filter. The returned map is keyed by media_items.content_id.
func (r *Repo) FilterAccessibleItemIDs(ctx context.Context, itemIDs []string, filter catalog.AccessFilter) (map[string]struct{}, error) {
if len(itemIDs) == 0 {
return map[string]struct{}{}, nil
}
conditions := []string{"mi.content_id = ANY($1)"}
args := []any{itemIDs}
argIdx := 2
if filter.AllowedContentIDs != nil {
if len(filter.AllowedContentIDs) == 0 {
return map[string]struct{}{}, nil
}
conditions = append(conditions, fmt.Sprintf("mi.content_id = ANY($%d)", argIdx))
args = append(args, filter.AllowedContentIDs)
argIdx++
}
if filter.AllowedLibraryIDs != nil {
if len(filter.AllowedLibraryIDs) == 0 {
return map[string]struct{}{}, nil
}
conditions = append(conditions, fmt.Sprintf(`
EXISTS (
SELECT 1
FROM media_item_libraries mil
WHERE mil.content_id = mi.content_id
AND mil.media_folder_id = ANY($%d)
)`, argIdx))
args = append(args, filter.AllowedLibraryIDs)
argIdx++
} else if len(filter.DisabledLibraryIDs) > 0 {
conditions = append(conditions, fmt.Sprintf(`
EXISTS (
SELECT 1
FROM media_item_libraries mil
WHERE mil.content_id = mi.content_id
AND mil.media_folder_id != ALL($%d)
)`, argIdx))
args = append(args, filter.DisabledLibraryIDs)
argIdx++
}
if filter.MaxContentRating != "" {
allowedRatings := access.AllowedRatingsUpTo(filter.MaxContentRating)
if len(allowedRatings) == 0 {
return map[string]struct{}{}, nil
}
conditions = append(conditions, fmt.Sprintf("mi.content_rating = ANY($%d)", argIdx))
args = append(args, allowedRatings)
argIdx++
}
rows, err := r.pool.Query(ctx, fmt.Sprintf(`
SELECT mi.content_id
FROM media_items mi
WHERE %s`, strings.Join(conditions, " AND ")), args...)
if err != nil {
return nil, fmt.Errorf("filter accessible item IDs: %w", err)
}
defer rows.Close()
result := make(map[string]struct{}, len(itemIDs))
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan accessible item ID: %w", err)
}
result[id] = struct{}{}
}
return result, rows.Err()
}
// GetItemAddedDates returns the created_at dates for a set of items.
func (r *Repo) GetItemAddedDates(ctx context.Context, itemIDs []string) (map[string]time.Time, error) {
if len(itemIDs) == 0 {
return nil, nil
}
rows, err := r.pool.Query(ctx, `
SELECT content_id, created_at
FROM media_items
WHERE content_id = ANY($1)`,
itemIDs)
if err != nil {
return nil, fmt.Errorf("get item added dates: %w", err)
}
defer rows.Close()
result := make(map[string]time.Time)
for rows.Next() {
var id string
var t time.Time
if err := rows.Scan(&id, &t); err != nil {
return nil, fmt.Errorf("scan item added date: %w", err)
}
result[id] = t
}
return result, rows.Err()
}
// GetAllUsersWithTasteProfiles returns all user/profile pairs that have taste profiles.
func (r *Repo) GetAllUsersWithTasteProfiles(ctx context.Context) ([]StaleProfile, error) {
rows, err := r.pool.Query(ctx, `SELECT user_id, profile_id FROM user_taste_profiles`)
if err != nil {
return nil, fmt.Errorf("get all users with taste profiles: %w", err)
}
defer rows.Close()
var profiles []StaleProfile
for rows.Next() {
var p StaleProfile
if err := rows.Scan(&p.UserID, &p.ProfileID); err != nil {
return nil, fmt.Errorf("scan user with taste profile: %w", err)
}
profiles = append(profiles, p)
}
return profiles, rows.Err()
}
// GetWatchedItemIDs returns content IDs of media items the user has watched
// (>= 50% progress or completed). Episodes are resolved to their parent series.
func (r *Repo) GetWatchedItemIDs(ctx context.Context, userID int, profileID string) ([]string, error) {
query := fmt.Sprintf(`
WITH %s
SELECT DISTINCT item_id
FROM watched_activity
WHERE user_id = $1 AND profile_id = $2`, watchedActivityCTE)
rows, err := r.pool.Query(ctx, query, userID, profileID)
if err != nil {
return nil, fmt.Errorf("get watched item IDs: %w", err)
}
defer rows.Close()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan watched item ID: %w", err)
}
ids = append(ids, id)
}
return ids, rows.Err()
}
// GetWatchedItemIDSet returns the watched item IDs as a set keyed by content ID.
func (r *Repo) GetWatchedItemIDSet(ctx context.Context, userID int, profileID string) (map[string]struct{}, error) {
ids, err := r.GetWatchedItemIDs(ctx, userID, profileID)
if err != nil {
return nil, err
}
return scoredItemIDSet(ids), nil
}
// GetWatchedItemIDSetFromStore derives watched item IDs from a user store,
// then canonicalizes episode progress rows to their parent series IDs.
func (r *Repo) GetWatchedItemIDSetFromStore(ctx context.Context, store userstore.UserStore, profileID string) (map[string]struct{}, error) {
if store == nil {
return map[string]struct{}{}, nil
}
const pageSize = 1000
rawIDs := make([]string, 0, pageSize)
offset := 0
for {
progress, err := store.ListProgress(ctx, profileID, "all", pageSize, offset)
if err != nil {
return nil, fmt.Errorf("list progress from store: %w", err)
}
for _, wp := range progress {
if wp.Completed || (wp.DurationSeconds > 0 && wp.PositionSeconds/wp.DurationSeconds >= 0.5) {
rawIDs = append(rawIDs, wp.MediaItemID)
}
}
if len(progress) < pageSize {
break
}
offset += len(progress)
}
return r.ResolveCanonicalItemIDSet(ctx, rawIDs)
}
// GetRecentCompletedItemIDs returns the most recently completed leaf item IDs for a profile.
func (r *Repo) GetRecentCompletedItemIDs(ctx context.Context, userID int, profileID string, limit int) ([]string, error) {
if limit <= 0 {
return []string{}, nil
}
query := fmt.Sprintf(`
WITH %s
SELECT leaf_item_id
FROM watched_activity
WHERE user_id = $1 AND profile_id = $2 AND completed = true
ORDER BY updated_at DESC, leaf_item_id ASC
LIMIT $3
`, watchedActivityCTE)
rows, err := r.pool.Query(ctx, query, userID, profileID, limit)
if err != nil {
return nil, fmt.Errorf("get recent completed item IDs: %w", err)
}
defer rows.Close()
ids := make([]string, 0, limit)
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, fmt.Errorf("scan recent completed item ID: %w", err)
}
ids = append(ids, id)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate recent completed item IDs: %w", err)
}
return ids, nil
}
// ExcludeWatchedItems removes watched items from a scored recommendation list.
func (r *Repo) ExcludeWatchedItems(ctx context.Context, userID int, profileID string, items []ScoredItem) ([]ScoredItem, error) {
if len(items) == 0 {
return items, nil
}
watchedSet, err := r.GetWatchedItemIDSet(ctx, userID, profileID)
if err != nil {
return nil, fmt.Errorf("get watched item IDs: %w", err)
}
return excludeScoredItems(items, watchedSet), nil
}
func scoredItemIDSet(ids []string) map[string]struct{} {
set := make(map[string]struct{}, len(ids))
for _, id := range ids {
set[id] = struct{}{}
}
return set
}
func excludeScoredItems(items []ScoredItem, excluded map[string]struct{}) []ScoredItem {
if len(items) == 0 || len(excluded) == 0 {
return items
}
filtered := make([]ScoredItem, 0, len(items))
for _, item := range items {
if _, ok := excluded[item.MediaItemID]; ok {
continue
}
filtered = append(filtered, item)
}
return filtered
}
// ResolveCanonicalItemIDSet maps episode IDs to their parent series IDs and
// leaves movie/series IDs unchanged.
func (r *Repo) ResolveCanonicalItemIDSet(ctx context.Context, itemIDs []string) (map[string]struct{}, error) {
if len(itemIDs) == 0 {
return map[string]struct{}{}, nil
}
set := scoredItemIDSet(itemIDs)
rows, err := r.pool.Query(ctx, `
SELECT content_id, series_id
FROM episodes
WHERE content_id = ANY($1)
`, itemIDs)
if err != nil {
return nil, fmt.Errorf("resolve canonical item IDs: %w", err)
}
defer rows.Close()
for rows.Next() {
var contentID string
var seriesID string
if err := rows.Scan(&contentID, &seriesID); err != nil {
return nil, fmt.Errorf("scan canonical item ID: %w", err)
}
delete(set, contentID)
set[seriesID] = struct{}{}
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate canonical item IDs: %w", err)
}
return set, nil
}
// CleanOldCacheTypes removes V1 cache entries that are no longer used.
func (r *Repo) CleanOldCacheTypes(ctx context.Context, userID int, profileID string) error {
_, err := r.pool.Exec(ctx,
`DELETE FROM recommendation_cache WHERE user_id = $1 AND profile_id = $2 AND rec_type IN ('for_you', 'taste_match')`,
userID, profileID)
if err != nil {
return fmt.Errorf("clean old cache types: %w", err)
}
return nil
}
// GetItemAllGenres returns the full genre array for each item ID.
func (r *Repo) GetItemAllGenres(ctx context.Context, itemIDs []string) (map[string][]string, error) {
if len(itemIDs) == 0 {
return nil, nil
}
rows, err := r.pool.Query(ctx, `
SELECT content_id, genres
FROM media_items
WHERE content_id = ANY($1)
AND genres IS NOT NULL
AND array_length(genres, 1) > 0`,
itemIDs)
if err != nil {
return nil, fmt.Errorf("get item genres: %w", err)
}
defer rows.Close()
result := make(map[string][]string)
for rows.Next() {
var id string
var genres []string
if err := rows.Scan(&id, &genres); err != nil {
return nil, fmt.Errorf("scan item genres: %w", err)
}
result[id] = genres
}
return result, rows.Err()
}
func (r *Repo) GetItemMediaTypes(ctx context.Context, itemIDs []string) (map[string]string, error) {
if len(itemIDs) == 0 {
return map[string]string{}, nil
}
rows, err := r.pool.Query(ctx, `
SELECT content_id, type
FROM media_items
WHERE content_id = ANY($1)`,
itemIDs)
if err != nil {
return nil, fmt.Errorf("get item media types: %w", err)
}
defer rows.Close()
result := make(map[string]string, len(itemIDs))
for rows.Next() {
var id, mediaType string
if err := rows.Scan(&id, &mediaType); err != nil {
return nil, fmt.Errorf("scan item media type: %w", err)
}
result[id] = mediaType
}
return result, rows.Err()
}
// ItemMetadata holds lightweight metadata used by the validation pipeline.
type ItemMetadata struct {
Title string
Type string
Genres []string
Year int
Studios []string
}
// GetItemMetadata returns lightweight metadata for a single item.
func (r *Repo) GetItemMetadata(ctx context.Context, itemID string) (*ItemMetadata, error) {
var m ItemMetadata
err := r.pool.QueryRow(ctx, `
SELECT title, COALESCE(type, ''), genres, COALESCE(year, 0), studios
FROM media_items WHERE content_id = $1`, itemID,
).Scan(&m.Title, &m.Type, &m.Genres, &m.Year, &m.Studios)
if err != nil {
return nil, fmt.Errorf("get item metadata %s: %w", itemID, err)
}
return &m, nil
}