haeaeae/silo-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/Silo-Server/silo-server

Browse Source 
# Conflicts:
#	cmd/silo/main.go
#	go.sum
#	internal/api/handlers/admin.go
#	internal/api/handlers/admin_jobs.go
#	internal/api/handlers/admin_jobs_test.go
#	internal/api/handlers/admin_test.go
#	internal/api/handlers/subtitle_search.go
#	internal/api/router.go
#	internal/auth/permissions.go
#	internal/auth/permissions_test.go
#	internal/auth/repository.go
#	internal/catalog/folder_delete_test.go
#	internal/catalog/folder_repo.go
#	internal/catalog/window_count_test.go
#	internal/metadata/worker.go
#	internal/requests/service.go
#	internal/scanner/file_repo.go
#	internal/scanqueue/repository.go
#	internal/scanqueue/service.go
#	internal/scantrigger/scantrigger.go
#	internal/scantrigger/scantrigger_test.go
#	internal/sections/defaults.go
#	internal/sections/defaults_test.go
#	web/src/api/client.ts
#	web/src/components/ContinueWatchingCard.tsx
#	web/src/components/ItemCard.tsx
#	web/src/lib/documentTitle.ts
#	web/src/lib/permissions.ts
#	web/src/pages/AdminDashboard.tsx
#	web/src/pages/AdminLibraries.test.tsx
#	web/src/pages/AdminLibraries.tsx
#	web/src/pages/AdminTasks.tsx
#	web/src/pages/AdminUserDetail.tsx
#	web/src/pages/AdminUsers.tsx
#	web/src/pages/ItemDetail/EpisodeContent.tsx
#	web/src/pages/ItemDetail/MovieContent.tsx
#	web/src/pages/ItemDetail/components/ActionBar.tsx
#	web/src/pages/LibraryBrowse.tsx
#	web/src/pages/LibraryPage.tsx
#	web/src/pages/RequestDetail.tsx
This commit is contained in:
Quick
2026-06-08 09:38:21 -04:00
parent cdb5b70a8d f5b3cba3cb
commit 3704b262d6
967 changed files with 105637 additions and 7528 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+22
View File
@@ -1,6 +1,12 @@
# Silo Docker deployment
# For the default docker compose setup, MEDIA_ROOT is the main value you need to set.
# Optional: layer compose files through env. Leave unset for CPU-only defaults.
# Linux/macOS example for enabling NVIDIA override:
# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
# Windows example:
# COMPOSE_FILE=docker-compose.yml;docker-compose.nvidia.yml
# Required for docker compose: host path to your media library.
# Example: MEDIA_ROOT=/srv/media
MEDIA_ROOT=/srv/media
@@ -20,6 +26,19 @@ SILO_IMAGE=ghcr.io/silo-server/silo-server:latest
POSTGRES_USER=silo
POSTGRES_PASSWORD=silo
POSTGRES_DB=silo
POSTGRES_SHM_SIZE=8gb
# PostgreSQL auto-tuning is enabled by default in docker-compose.yml. Silo
# applies pgtune-style OLTP recommendations through ALTER SYSTEM at startup.
# Set POSTGRES_TUNE=off if you manage PostgreSQL tuning yourself.
# POSTGRES_TUNE=off
# POSTGRES_TUNE_PROFILE=oltp
# POSTGRES_TUNE_MEMORY=auto
# POSTGRES_TUNE_MEMORY_BUDGET_PERCENT=75
# POSTGRES_TUNE_CPUS=auto
# POSTGRES_TUNE_STORAGE=ssd
# POSTGRES_TUNE_DB_SIZE=auto
# POSTGRES_TUNE_CONNECTIONS=100
# Optional host port overrides for docker-compose.yml.
# PORT=8090
@@ -27,6 +46,9 @@ POSTGRES_DB=silo
# PROXY_PORT=8083
# TRANSCODE_PORT=8082
# Optional NVIDIA override controls (used by docker-compose.nvidia.yml).
# NVIDIA_GPU_COUNT=1
# Run from source / advanced overrides
# Only DATABASE_URL is required when running Silo outside the default docker compose stack.
# DATABASE_URL=postgres://silo:password@localhost:5432/silo
AGENTS.md
+2 -1
View File
@@ -1,7 +1,7 @@
# Repository Guidelines
## Project Structure & Module Organization
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/` as paired numbered `.up.sql` and `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/sql/` as Goose SQL migrations. Legacy converted migrations intentionally keep their original numeric versions so existing `schema_versions` rows can bootstrap cleanly into Goose. New migrations must use timestamped filenames created with `make migrate-create NAME=add_thing`; do not run `goose fix` or create paired `.up.sql` / `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
When creating or editing `docs/superpowers/specs/` or `docs/superpowers/plans/`, never include local absolute filesystem paths or transient worktree IDs. Use repository-relative paths and wording like "Commands assume the repository root is the cwd."
@@ -53,6 +53,7 @@ Use the checked-in `Makefile` for the common paths:
- `make dev-frontend`: start the Vite dev server with HMR
- `make dev-proxy` / `make dev-transcode`: run standalone worker modes
- `make lint`: run `golangci-lint` and frontend ESLint
- `make migrate-status` / `make migrate-up`: inspect or apply Goose migrations through Silo's legacy-safe bootstrapping runner
Run before opening a merge request:
Dockerfile
+2 -2
View File
@@ -48,13 +48,13 @@ RUN apt-get update && \
echo "deb [signed-by=/usr/share/keyrings/jellyfin.gpg arch=${TARGETARCH}] https://repo.jellyfin.org/debian bookworm main" \
> /etc/apt/sources.list.d/jellyfin.list && \
apt-get update && \
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 && \
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 fonts-noto-core fonts-noto-cjk && \
apt-get purge -y gnupg && apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
RUN mkdir -p /tmp/silo-transcode
COPY --from=build /silo /usr/local/bin/silo
COPY third_party/jellyfin-web/ /srv/jellyfin-web/
EXPOSE 8080 8096
EXPOSE 8080 8096 13378
HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \
CMD curl -f http://localhost:${PORT:-8080}/api/v1/health || exit 1
ENTRYPOINT ["silo"]
Dockerfile.dev
+1 -1
View File
@@ -53,7 +53,7 @@ RUN apt-get update && \
echo "deb [signed-by=/usr/share/keyrings/jellyfin.gpg arch=${TARGETARCH}] https://repo.jellyfin.org/debian bookworm main" \
> /etc/apt/sources.list.d/jellyfin.list && \
apt-get update && \
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 && \
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 fonts-noto-core fonts-noto-cjk && \
apt-get purge -y gnupg && apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
RUN mkdir -p /tmp/silo-transcode
Makefile
+21 -1
View File
@@ -1,10 +1,13 @@
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks migrate-create migrate-validate migrate-status migrate-up
GIT_COMMON_DIR := $(strip $(shell git rev-parse --git-common-dir 2>/dev/null))
MAIN_CHECKOUT_ROOT := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/..))
SHARED_MAKEFILE_LOCAL := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/../Makefile.local))
DEFAULT_PLUGIN_SDK_DIR := $(abspath ../silo-plugin-sdk)
SHARED_PLUGIN_SDK_DIR := $(if $(MAIN_CHECKOUT_ROOT),$(abspath $(MAIN_CHECKOUT_ROOT)/../silo-plugin-sdk))
GOOSE := go run github.com/pressly/goose/v3/cmd/goose@v3.27.1
GOOSE_DIR := migrations/sql
ENV_FILE ?= .env
ifneq ($(wildcard $(DEFAULT_PLUGIN_SDK_DIR)),)
DEV_PLUGIN_SDK_DIR ?= $(DEFAULT_PLUGIN_SDK_DIR)
@@ -54,6 +57,23 @@ lint:
verify-local-paths:
scripts/check-local-path-leaks.sh
# Create a timestamped Goose SQL migration. Usage: make migrate-create NAME=add_thing
migrate-create:
@if [ -z "$(NAME)" ]; then echo "usage: make migrate-create NAME=add_thing"; exit 1; fi
$(GOOSE) -dir $(GOOSE_DIR) create "$(NAME)" sql
# Validate Goose migration annotations and SQL parsing without touching a database.
migrate-validate:
$(GOOSE) -dir $(GOOSE_DIR) validate
# Show Goose migration status through Silo's bootstrapping runner.
migrate-status:
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-status
# Apply pending Goose migrations through Silo's bootstrapping runner.
migrate-up:
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-only
# Install repo-local git hooks for this checkout/worktree.
install-hooks:
@existing="$$(git config --local core.hooksPath 2>/dev/null || true)"; \
README.md
+97 -80
View File
@@ -2,6 +2,8 @@
A self-hosted media streaming server with a React frontend and Go backend. Supports direct play, remuxing, and hardware-accelerated transcoding. Compatible with Jellyfin/Emby clients (VidHub, Findroid) via a built-in compatibility layer.
Join the community on [Discord](https://discord.gg/4RxuUQAEnW).
## Deploy with Docker (recommended)
The easiest way to run Silo is with Docker Compose. The default stack assumes you do not already have PostgreSQL and Redis available, so it bundles PostgreSQL, Redis, FFmpeg, and the application for a one-command start.
@@ -32,6 +34,29 @@ The easiest way to run Silo is with Docker Compose. The default stack assumes yo
If you already have PostgreSQL and Redis available, omit those bundled service examples from compose and point Silo at your existing `DATABASE_URL` and `REDIS_URL` instead.
### Optional NVIDIA/NVENC
GPU support is kept out of the default compose file so hosts without NVIDIA drivers work unchanged.
Install the NVIDIA Container Toolkit and use a Docker Compose version with GPU reservation support before enabling this override.
Use the optional override file when you want NVENC:
```sh
docker compose -f docker-compose.yml -f docker-compose.nvidia.yml up -d
```
If you want this controlled from `.env`, set `COMPOSE_FILE`:
```dotenv
COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
NVIDIA_GPU_COUNT=1
```
Windows uses `;` instead of `:` between compose files.
Then `docker compose up -d` will include the NVIDIA override automatically.
4. **Configure through the admin UI**
Add libraries, users, metadata providers, and playback settings from the web interface.
@@ -176,8 +201,36 @@ Technical notes:
| `make dev-backend` | Run Go backend (integrated mode) |
| `make dev-proxy` | Run a standalone proxy node |
| `make dev-transcode` | Run a standalone transcode node |
| `make migrate-create NAME=add_thing` | Create a timestamped Goose SQL migration |
| `make migrate-validate` | Validate Goose migration files without touching a database |
| `make migrate-status` | Show Goose migration status using Silo's bootstrapping runner |
| `make migrate-up` | Apply pending Goose migrations using Silo's bootstrapping runner |
| `make clean` | Remove build artifacts |
### Database Migrations
PostgreSQL schema migrations are managed by Goose. Migration SQL files live in
`migrations/sql/` and use Goose annotations. Converted legacy migrations keep
their original numeric versions so existing `schema_versions` rows can bootstrap
cleanly into Goose without replaying old SQL. New migrations should be created
with timestamped filenames:
```sh
make migrate-create NAME=add_thing
make migrate-validate
```
Do not run `goose fix`; timestamped migrations are the repository policy because
they avoid version collisions across parallel PRs. The existing `001`-style
files are historical compatibility records, not the naming pattern for new work.
Runtime migrations are applied by the integrated/API server only. Proxy and
transcode modes never mutate schema.
For existing installs, use `make migrate-status` and `make migrate-up` rather
than invoking the Goose CLI directly; those targets copy legacy
`schema_versions` rows into `public.goose_db_version` under the migration lock
before reading or applying migrations. Set `ENV_FILE=path/to/.env` when the
database URL should be read from a non-default env file.
### Running Tests
```sh
@@ -216,93 +269,57 @@ Silo requires only a `DATABASE_URL` when running from source or against external
| `proxy` | Stream proxy node that connects to the shared deployment database and Redis |
| `transcode` | HLS transcode worker node that connects to the shared deployment database and Redis |
## PostgreSQL Performance Tuning
## PostgreSQL Auto-Tuning
The default PostgreSQL configuration is conservative and not optimized for any particular workload. For better performance, use [PGTune](https://pgtune.leopard.in.ua/) to generate settings tailored to your hardware.
**Recommended PGTune settings:**
| Setting | Value |
|---|---|
| DB Type | **Web application** |
| Total Memory | Your server's RAM |
| Number of CPUs | Your server's CPU count |
| Number of Connections | 20 (Silo's default pool size) |
| Data Storage | SSD or HDD depending on your setup |
Silo's workload is predominantly read-heavy API queries (browse, search, metadata lookups) with the database small enough to fit in RAM for most libraries. The **Web application** profile optimizes for this: CPU-bound simple queries served from cached pages, high concurrent reads, and low write volume.
If you have a very large library (50,000+ items) or heavy activity logging, consider switching to the **OLTP** profile instead.
### Recommended Defaults for Modern Hardware
Assuming you're using modern hardware with fast NVMe storage this is what I'm currently running.
```ini
# Connections
max_connections = 100
# Memory (tuned to keep PG under ~8GB total)
shared_buffers = 4GB # holds the entire DB in cache (typical DB is 1-4GB)
effective_cache_size = 6GB # planner hint only, not an allocation
work_mem = 32MB # per-operation sort/hash memory
maintenance_work_mem = 512MB # VACUUM, CREATE INDEX, catalog imports
# WAL / Write Performance
wal_buffers = 64MB
min_wal_size = 512MB
max_wal_size = 2GB
checkpoint_completion_target = 0.9
# Storage (NVMe/Optane)
random_page_cost = 1.1 # default 4.0; NVMe random ≈ sequential
effective_io_concurrency = 200 # NVMe handles many concurrent requests
# Parallelism
max_worker_processes = 8
max_parallel_workers_per_gather = 4
max_parallel_workers = 8
max_parallel_maintenance_workers = 4
# Huge Pages
huge_pages = try
```
These settings keep PostgreSQL under ~8GB of RAM while still caching the entire database. For HDD storage, change `random_page_cost` to `4.0` and `effective_io_concurrency` to `2`.
### Applying Settings
**Docker (recommended):** Save the settings above to a `postgresql.conf` file and mount it into your container:
The default Docker Compose stack does not require a checked-in `postgresql.conf`.
It enables Silo's [pgtune](https://github.com/le0pard/pgtune)-style OLTP tuning
by default:
```yaml
# docker-compose.yml
postgres:
image: pgvector/pgvector:pg18
volumes:
- ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
command: postgres -c config_file=/etc/postgresql/postgresql.conf
POSTGRES_TUNE: auto
```
Then restart: `docker compose restart postgres`
When enabled, Silo connects with `DATABASE_URL` and applies recommendations with
`ALTER SYSTEM`, which writes to PostgreSQL's `postgresql.auto.conf` inside the
database data directory. Reloadable settings are applied immediately with
`pg_reload_conf()`. Settings that PostgreSQL marks as restart-only are written
too, and Silo logs the setting names so you can restart PostgreSQL once:
**Existing PostgreSQL:** You can apply settings without editing config files using `ALTER SYSTEM`, which writes to `postgresql.auto.conf` inside the data directory:
```sql
ALTER SYSTEM SET work_mem = '32MB';
ALTER SYSTEM SET effective_cache_size = '6GB';
ALTER SYSTEM SET random_page_cost = 1.1;
ALTER SYSTEM SET effective_io_concurrency = 200;
-- Apply without restart:
SELECT pg_reload_conf();
-- These require a full restart:
ALTER SYSTEM SET shared_buffers = '4GB';
ALTER SYSTEM SET max_connections = 100;
ALTER SYSTEM SET max_worker_processes = 8;
ALTER SYSTEM SET huge_pages = 'try';
```sh
docker compose restart postgres
```
Settings like `shared_buffers`, `max_connections`, `max_worker_processes`, and `huge_pages` require a PostgreSQL restart. Memory/planner settings (`work_mem`, `effective_cache_size`, `random_page_cost`) can be reloaded live.
The default Compose database user has the required PostgreSQL permissions. If
you use an external PostgreSQL server, make sure the configured `DATABASE_URL`
user can run `ALTER SYSTEM`, or set `POSTGRES_TUNE=off` and manage
PostgreSQL yourself.
For `POSTGRES_TUNE_MEMORY=auto`, Silo uses the first trustworthy memory source:
a finite Docker cgroup limit, the read-only `/host/proc/meminfo` mount supplied
by the bundled Compose file, then `/proc/meminfo` with container safety guards.
Auto-detected memory is treated as a PostgreSQL budget, defaulting to 75% of
detected RAM so Silo, Redis, plugins, transcodes, and the OS retain headroom.
`POSTGRES_TUNE_DB_SIZE=auto` queries `pg_database_size(current_database())` and
classifies the workload by comparing the database size to that memory budget.
Optional tuning overrides:
| Variable | Default | Description |
|---|---:|---|
| `POSTGRES_TUNE_PROFILE` | `oltp` | Tuning profile. Only `oltp` is currently supported. |
| `POSTGRES_TUNE_MEMORY` | `auto` | Server/container RAM, such as `8GB` or `32GB`; explicit values are used as-is. |
| `POSTGRES_TUNE_MEMORY_BUDGET_PERCENT` | `75` | Percent of auto-detected RAM used for PostgreSQL recommendations. |
| `POSTGRES_TUNE_CPUS` | `auto` | CPU count used for worker recommendations. |
| `POSTGRES_TUNE_STORAGE` | `ssd` | One of `hdd`, `ssd`, `san`, or `nvme`. |
| `POSTGRES_TUNE_DB_SIZE` | `auto` | Use `less_ram` when the database comfortably fits in RAM, `mid_ram`, or `greater_ram` for very large databases. |
| `POSTGRES_TUNE_CONNECTIONS` | `100` | PostgreSQL `max_connections`; automatically raised if Silo's app pool is configured higher. |
| `POSTGRES_SHM_SIZE` | `8gb` | Docker `/dev/shm` size for the bundled PostgreSQL container. |
Advanced operators can still supply their own PostgreSQL configuration or
override these env vars. Set `POSTGRES_TUNE=off` when you do not want Silo to
change PostgreSQL server settings. Settings already written with `ALTER SYSTEM`
remain in `postgresql.auto.conf`; reset those PostgreSQL parameters if you later
move fully to a custom `postgresql.conf`.
## Project Structure
@@ -319,5 +336,5 @@ internal/
scanner/ Media file discovery and FFProbe
worker/ Background jobs (scan, match, reconcile)
web/ React + TypeScript frontend (Vite, Tailwind, shadcn/ui)
migrations/ PostgreSQL schema migrations
migrations/sql/ Goose-managed PostgreSQL schema migrations
```
cmd/silo/main.go
+548 -73
View File
@@ -4,8 +4,6 @@ import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"io/fs"
@@ -16,24 +14,32 @@ import (
"os/signal"
"path/filepath"
"runtime/debug"
"sort"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/go-chi/chi/v5"
chimiddleware "github.com/go-chi/chi/v5/middleware"
"github.com/google/uuid"
"github.com/hashicorp/go-hclog"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/prometheus/client_golang/prometheus/promhttp"
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
sdkcapability "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability"
"github.com/Silo-Server/silo-server/internal/access"
"github.com/Silo-Server/silo-server/internal/activitylog"
"github.com/Silo-Server/silo-server/internal/adminjob"
"github.com/Silo-Server/silo-server/internal/api"
"github.com/Silo-Server/silo-server/internal/api/handlers"
"github.com/Silo-Server/silo-server/internal/audiobooks"
"github.com/Silo-Server/silo-server/internal/audiobooks/podcastfeed"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/autoscan"
"github.com/Silo-Server/silo-server/internal/cache"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/catalogseed"
@@ -50,7 +56,6 @@ import (
"github.com/Silo-Server/silo-server/internal/logfilter"
"github.com/Silo-Server/silo-server/internal/logstream"
"github.com/Silo-Server/silo-server/internal/markers"
"github.com/Silo-Server/silo-server/internal/markers/introdb"
"github.com/Silo-Server/silo-server/internal/mdblist"
"github.com/Silo-Server/silo-server/internal/metadata"
"github.com/Silo-Server/silo-server/internal/models"
@@ -183,8 +188,74 @@ func configureOperationalLogging(
return operationalWriter, opslog.NewRepo(pool), opsPM
}
func maybeApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, appMaxConnections int, mode string) {
switch strings.ToLower(strings.TrimSpace(mode)) {
case "", "integrated", "api":
default:
return
}
opts, err := database.LoadPostgresTuneOptionsFromEnv(appMaxConnections)
if err != nil {
slog.Warn("postgres auto-tuning disabled", "error", err)
return
}
if !opts.Enabled {
return
}
tuneCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
result, err := database.ApplyPostgresTuning(tuneCtx, pool, opts)
for _, failure := range result.Failures {
slog.Warn("postgres auto-tuning setting failed",
"name", failure.Name,
"value", failure.Value,
"error", failure.Err,
)
}
if err != nil {
slog.Warn("postgres auto-tuning failed",
"error", err,
"applied", result.Applied,
"failures", len(result.Failures),
)
return
}
slog.Info("postgres auto-tuning applied",
"profile", opts.Profile,
"postgres_major", result.PostgresMajorVersion,
"settings", result.Applied,
"resets", len(result.Reset),
"failures", len(result.Failures),
"memory_budget_bytes", opts.MemoryBudgetBytes,
"detected_memory_bytes", opts.DetectedMemoryBytes,
"memory_source", opts.MemorySource,
"memory_budget_percent", opts.MemoryBudgetPercent,
"cpus", opts.CPUs,
"connections", opts.Connections,
"storage", opts.Storage,
"db_size", result.DBSize,
"database_size_bytes", result.DatabaseSizeBytes,
)
if len(result.RestartRequired) > 0 {
slog.Warn("postgres restart required to finish applying auto-tuned settings",
"settings", strings.Join(result.RestartRequired, ","),
)
}
if len(result.Reset) > 0 {
slog.Info("postgres auto-tuning reset stale settings",
"settings", strings.Join(result.Reset, ","),
)
}
}
func main() {
envFile := flag.String("env", ".env", "path to .env bootstrap file")
migrateOnly := flag.Bool("migrate-only", false, "apply database migrations and exit")
migrateStatus := flag.Bool("migrate-status", false, "show database migration status and exit")
flag.Parse()
ctx := context.Background()
@@ -204,12 +275,48 @@ func main() {
defer pool.Close()
slog.Info("connected to PostgreSQL")
if *migrateStatus {
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
statuses, statusErr := database.MigrationStatuses(migCtx, pool, migrations.FS, "sql")
migCancel()
if statusErr != nil {
log.Fatalf("failed to read migration status: %v", statusErr)
}
fmt.Printf("%-8s %8s %-25s %s\n", "STATE", "VERSION", "APPLIED_AT", "MIGRATION")
for _, status := range statuses {
appliedAt := "-"
if !status.AppliedAt.IsZero() {
appliedAt = status.AppliedAt.UTC().Format(time.RFC3339)
}
source := status.Source
if source != "" {
source = filepath.Base(source)
} else {
source = "-"
}
fmt.Printf("%-8s %8d %-25s %s\n", status.State, status.Version, appliedAt, source)
}
return
}
if *migrateOnly {
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql")
migCancel()
if migErr != nil {
log.Fatalf("failed to run migrations: %v", migErr)
}
slog.Info("database migrations applied")
return
}
// Run migrations only for integrated/api modes. Proxy and transcode nodes
// should never alter the schema — they may scale independently and would
// race or apply migrations before the primary node is deliberately upgraded.
if bc.Mode == "integrated" || bc.Mode == "api" || bc.Mode == "" {
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "."); migErr != nil {
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql"); migErr != nil {
migCancel()
log.Fatalf("failed to run migrations: %v", migErr)
}
@@ -313,10 +420,13 @@ func main() {
slog.SetDefault(slog.New(logfilter.New(baseHandler, cfg.Server.LogQuiet)))
mode := cfg.Server.Mode
maybeApplyPostgresTuning(ctx, pool, cfg.Database.MaxConnections, mode)
slog.Info("silo starting", "mode", mode, "listen", cfg.Server.Listen, "log_level", cfg.Server.LogLevel, "node_id", nodeID)
appCtx, appCancel := context.WithCancel(ctx)
defer appCancel()
restartReqCh := make(chan struct{}, 1)
var restartRequested atomic.Bool
eventBus := cache.NewEventBus(cfg.Redis.URL)
logStreamHub := logstream.NewHub(nodeID, eventBus)
@@ -417,7 +527,22 @@ func main() {
OpsLogRepo: opsRepo,
FFmpegLogSink: playback.NewSlogFFmpegLogSink(slog.Default(), nodeID),
PublicURL: os.Getenv("SILO_PUBLIC_URL"),
RequestServerRestart: func(context.Context) error {
if !restartRequested.CompareAndSwap(false, true) {
return handlers.ErrServerRestartAlreadyRequested
}
restartReqCh <- struct{}{}
return nil
},
}
audiobooksService := audiobooks.New(&audiobooksSettingsAdapter{repo: settingsRepo})
absCompatEnabled, err := audiobooksService.ABSCompatEnabled(appCtx)
if err != nil {
slog.Warn("Audiobookshelf compatibility disabled; failed to read setting", "err", err)
absCompatEnabled = false
}
adminJobCancelRegistry := adminjob.NewCancelRegistry()
deps.AdminJobCancelRegistry = adminJobCancelRegistry
if needsWorkers && deps.DB != nil {
deps.IntroRepository = intromarkers.NewRepository(deps.DB)
deps.IntroAnalyzer = intromarkers.NewAnalyzer(
@@ -428,36 +553,34 @@ func main() {
}
if deps.DB != nil {
markerRegistry := markers.NewRegistry(slog.Default())
introdbAPIKey, err := settingsRepo.Get(appCtx, "introdb.api_key")
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
slog.Warn("load introdb.api_key from settings failed; provider will run with no key",
markerProviderConfig := markers.NewProviderConfigStore(deps.DB)
if err := markerProviderConfig.Reload(appCtx); err != nil {
slog.Warn("load marker provider config failed; falling back to registration-order fetch",
"error", err)
introdbAPIKey = ""
}
introdbClient := introdb.NewClient(introdbAPIKey)
if err := markerRegistry.Register(introdb.NewProvider(introdbClient)); err != nil {
log.Fatalf("register introdb marker provider: %v", err)
}
deps.OnServerSettingUpdated = func(_ context.Context, key, value string) {
if key == "introdb.api_key" {
introdbClient.SetAPIKey(value)
} else {
markerRegistry.UseConfigStore(markerProviderConfig)
if deps.EventBus != nil {
if err := deps.EventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
if event.Type != cache.EventMarkerProviderConfigChanged {
return
}
if err := markerProviderConfig.Reload(appCtx); err != nil {
slog.Warn("reload marker provider config failed", "provider", event.Payload, "error", err)
}
}); err != nil {
slog.Warn("subscribe marker provider config reload failed", "error", err)
}
}
}
if eventBus != nil {
_ = eventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
if event.Type != cache.EventSettingsChanged || event.Payload != "introdb.api_key" {
return
}
value, loadErr := settingsRepo.Get(context.Background(), "introdb.api_key")
if loadErr != nil {
slog.Warn("introdb api key reload failed", "error", loadErr)
return
}
introdbClient.SetAPIKey(value)
})
}
deps.MarkerProviderConfig = markerProviderConfig
deps.MarkerRegistry = markerRegistry
deps.MarkerResolver = markers.NewDBExternalIDResolver(deps.DB)
markerResolver := markers.NewDBExternalIDResolver(deps.DB)
deps.MarkerResolver = markerResolver
markerContributionStore := markers.NewContributionStore(deps.DB)
deps.MarkerContributionStore = markerContributionStore
deps.MarkerContributionService = markers.NewContributionService(
markerRegistry, markerResolver, markerProviderConfig, markerContributionStore, slog.Default(),
)
}
var watchProviderService *watchsync.Service
if deps.DB != nil {
@@ -669,6 +792,22 @@ func main() {
installer,
plugins.NewHostAdapter(pluginHost),
)
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
markerPluginResolver := markers.NewPluginResolverAdapter(pluginService)
pluginService.AddLifecycleHook(func(ctx context.Context) {
if err := reloadMarkerPluginProviders(
ctx,
deps.MarkerRegistry,
deps.MarkerProviderConfig,
installationStore,
runtimeConfigStore,
settingsRepo,
markerPluginResolver,
); err != nil {
slog.Warn("reload marker plugin providers failed", "error", err)
}
})
}
if err := pluginService.PreloadEnabled(appCtx); err != nil {
log.Fatalf("preload enabled plugins: %v", err)
}
@@ -748,6 +887,7 @@ func main() {
var groupClaimRepo *catalog.GroupClaimRepository
var seasonRepo *catalog.SeasonRepository
var episodeRepo *catalog.EpisodeRepository
var audiobookEnricher *audiobooks.Enricher
if needsWorkers && deps.DB != nil && deps.FileRepo != nil {
chainRepo := metadata.NewChainRepository(deps.DB)
skippedRootRepo = metadata.NewSkippedRootRepository(deps.DB)
@@ -803,6 +943,8 @@ func main() {
providerIDRepo := catalog.NewProviderIDRepository(deps.DB)
movieQueueRepo = metadata.NewMovieMatchQueueRepository(deps.DB, deps.FileRepo)
seriesQueueRepo = metadata.NewSeriesRootMatchQueueRepository(deps.DB)
deps.MovieMatchQueueRepo = movieQueueRepo
deps.SeriesRootMatchQueueRepo = seriesQueueRepo
matchQueueCoordinator = metadata.NewMatchQueueCoordinator(movieQueueRepo, seriesQueueRepo)
rootClaimRepo = catalog.NewRootClaimRepository(deps.DB)
groupClaimRepo = catalog.NewGroupClaimRepository(deps.DB)
@@ -816,6 +958,18 @@ func main() {
personRefreshService = metadata.NewPersonRefreshService(deps.DB, pluginResolver, personRepo)
personRefreshService.SetImageResolver(imageResolver)
// Wire the audiobook enricher. It uses the same plugin resolver and chain
// repo as the movie/TV pipeline, but resolves providers at
// content_level='audiobook' and sweeps items directly rather than via a queue.
audiobookEnricher = audiobooks.NewEnricher(
deps.DB,
chainRepo,
pluginResolver,
itemRepo,
personRepo,
providerIDRepo,
)
// Always wire the image resolver so plugin-prefixed URLs (e.g.
// metadb://) can be resolved to presigned HTTP URLs in API responses.
metadataService.SetImageResolver(imageResolver)
@@ -826,13 +980,21 @@ func main() {
imageCacher := imagecache.New(deps.S3Public)
metadataService.SetImageCacher(imageCacher)
metadataService.SetAutoCacheImages(cfg.Metadata.CacheImages)
if deps.Scanner != nil {
deps.Scanner.SetImageCacher(imageCacher)
}
if cfg.Metadata.CacheImages {
personRefreshService.SetImageCacher(imageCacher)
slog.Info("metadata image caching enabled")
}
if audiobookEnricher != nil {
audiobookEnricher.SetImageCacher(imageCacher)
audiobookEnricher.SetFFmpegPath(scanner.FFmpegPathFromFFprobe(scanner.FFprobePathFromFFmpeg(cfg.Playback.FFmpegPath)))
}
}
matchWorker = metadata.NewMatchWorker(metadataService, deps.FileRepo, cfg.Matcher.Workers, cfg.Matcher.BatchSize, 30*time.Second)
matchWorker.SetRealtimeHub(deps.RealtimeHub)
if movieQueueRepo != nil {
matchWorker.SetMovieFileClaimer(movieQueueRepo)
}
@@ -1281,9 +1443,15 @@ func main() {
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil {
taskMgr.Register(tasks.NewScanLibrariesTask(deps.FolderRepo, deps.LibraryScanQueue, deps.EventBus))
}
taskMgr.Register(tasks.NewCleanupOrphanedMediaItemsTask(catalog.NewOrphanedProvisionalCleaner(deps.DB)))
if deps.IntroAnalyzer != nil {
taskMgr.Register(tasks.NewDetectIntroMarkersTask(deps.IntroAnalyzer, settingsRepo))
}
if deps.MarkerContributionService != nil && deps.MarkerProviderConfig != nil && deps.MarkerContributionStore != nil && deps.FileRepo != nil {
taskMgr.Register(tasks.NewContributeMarkersTask(
deps.MarkerContributionService, deps.MarkerProviderConfig, deps.MarkerContributionStore, deps.FileRepo,
))
}
if chapterBackfiller, ok := deps.ChapterThumbnailQueuer.(*chapterthumbs.Service); ok {
taskMgr.Register(tasks.NewChapterThumbnailBackfillTask(chapterBackfiller, 25))
}
@@ -1320,13 +1488,52 @@ func main() {
)
requestReconcileSvc.SetSecretResolver(settingsRepo)
requestReconcileSvc.SetFulfillmentAdapters(radarr.NewClient(nil), sonarr.NewClient(nil))
if userStoreProvider != nil {
reconcileResolver := access.NewResolver(
auth.NewUserRepository(deps.DB),
userStoreProvider,
access.NewProfileTokenService(cfg.Auth.JWTSecret, 0),
)
requestReconcileSvc.SetEntitlementResolver(mediarequests.NewAccessEntitlements(reconcileResolver))
}
taskMgr.Register(tasks.NewReconcileRequestsTask(requestReconcileSvc, 100))
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil && pluginService != nil && pluginInstallationStore != nil {
autoscanRepo := autoscan.NewRepository(deps.DB)
if err := autoscanRepo.MarkInterruptedEvents(appCtx); err != nil {
slog.Warn("autoscan: failed to mark interrupted polls", "err", err)
}
autoscanSvc := api.BuildAutoscanService(
autoscanRepo,
pluginService,
pluginInstallationStore,
mediarequests.NewRepository(deps.DB),
settingsRepo,
deps.FolderRepo,
deps.LibraryScanQueue,
deps.RedisClient,
)
// The poll task's default interval seeds the schedule from the stored
// settings (DefaultPollIntervalSeconds); per-cycle gating still runs
// off the live settings inside PollOnce. Seed in MILLISECONDS as
// seconds*1000 — the SAME computation HandleUpdateSettings uses to
// reschedule — so startup and reschedule agree for sub-minute and
// non-60-multiple intervals (the old seconds/60 minutes path diverged).
var intervalMs int64 = 10 * 60 * 1000
if settings, serr := autoscanRepo.GetSettings(appCtx); serr == nil && settings.DefaultPollIntervalSeconds > 0 {
intervalMs = int64(settings.DefaultPollIntervalSeconds) * 1000
}
taskMgr.Register(tasks.NewAutoscanPollTask(autoscanSvc, intervalMs))
}
reconcileProviderIDRepo := catalog.NewProviderIDRepository(deps.DB)
reconcileEpisodeRepo := catalog.NewEpisodeRepository(deps.DB)
historyResolver := watchstate.NewStableIdentityResolver(nil, reconcileEpisodeRepo, reconcileProviderIDRepo)
historyReconciler := watchstate.NewHistoryReconciler(deps.DB, historyResolver)
taskMgr.Register(tasks.NewRepairProviderIDIntegrityTask(metadata.NewProviderIDIntegrityRepairer(deps.DB), historyReconciler))
taskMgr.Register(tasks.NewReconcileWatchHistoryTask(historyReconciler))
taskMgr.Register(tasks.NewSyncPodcastFeedsTask(podcastfeed.New(), podcastfeed.NewDBStore(deps.DB)))
if audiobookEnricher != nil {
taskMgr.Register(tasks.NewSyncAudiobookMetadataTask(audiobookEnricher))
}
if pluginInstallationStore != nil && pluginRuntimeConfigStore != nil && pluginService != nil {
pluginTasks, err := plugins.NewTaskRegistryWithTypedResolver(pluginInstallationStore, pluginRuntimeConfigStore, pluginService).Tasks(appCtx)
if err != nil {
@@ -1343,6 +1550,57 @@ func main() {
slog.Info("task manager started")
}
// Build the ABS-compatible REST + Socket.io handler when a DB pool is
// available. Routes are mounted at the root level by NewRouter (not under
// /api/v1/) so ABS clients resolve /login, /api/*, /abs/api/*, and
// /abs/socket.io/* without path prefix hacks.
if absCompatEnabled && deps.DB != nil {
absUserRepo := auth.NewUserRepository(deps.DB)
absSessionRepo := auth.NewSessionRepository(deps.DB)
absJWTService := auth.NewJWTService(
cfg.Auth.JWTSecret,
cfg.Auth.AccessTokenExpiry,
cfg.Auth.RefreshTokenExpiry,
)
absAuthSvc := auth.NewService(
auth.NewLocalProvider(absUserRepo, absSessionRepo),
absJWTService,
absSessionRepo,
absUserRepo,
nil, // invite codes: not needed for ABS compat
nil, // settings: not needed here
nil, // user store: not needed here
)
absItemRepo := catalog.NewItemRepository(deps.DB)
absEpisodeRepo := catalog.NewEpisodeRepository(deps.DB)
absSeasonRepo := catalog.NewSeasonRepository(deps.DB)
absPersonRepo := catalog.NewPersonRepository(deps.DB)
var absFileFetcher catalog.FileVersionFetcher
if deps.FileRepo != nil {
absFileFetcher = deps.FileRepo
}
absDetailSvc := catalog.NewDetailService(absItemRepo, absEpisodeRepo, absSeasonRepo, absPersonRepo, absFileFetcher)
if deps.ImageResolver != nil {
absDetailSvc.SetImageResolver(deps.ImageResolver)
}
absHDeps := audiobooks.ABSHandlerDeps{
Pool: deps.DB,
Items: absItemRepo,
Files: deps.FileRepo,
Settings: settingsRepo,
Auth: &audiobooks.SiloCredValidator{
Auth: absAuthSvc,
Pool: deps.DB,
},
AccessResolver: audiobooks.NewABSAccessResolver(absUserRepo, userStoreProvider),
Recs: recommendations.NewRepo(deps.DB),
Detail: absDetailSvc,
}
absH := audiobooksService.BuildABSHandler(absHDeps)
deps.ABSHandler = absH
}
_ = audiobooksService
if deps.DB != nil && pluginInstallationStore != nil && pluginRuntimeConfigStore != nil && deps.PluginService != nil {
userRepo := auth.NewUserRepository(deps.DB)
sessionRepo := auth.NewSessionRepository(deps.DB)
@@ -1464,6 +1722,11 @@ func main() {
metricsMux := http.NewServeMux()
metricsMux.Handle("/metrics", promhttp.Handler())
metricsMux.Handle("/api/", router)
// ABS-compat is NOT mounted on the main listener — see the "ABS compat
// listener" block below. It binds its own port so the discovery probes
// (/ping, /healthcheck, /status, /init, /login, /socket.io) own the URL
// space without collision with silo's SPA fallback. Mirrors how the
// Jellyfin compat server is set up at :8096.
metricsMux.Handle("/", server.FrontendHandler())
// Step 9: Start background workers (if needed).
@@ -1530,6 +1793,7 @@ func main() {
templateBundleApplyExecutor,
deps.RealtimeHub,
)
adminJobRunner.SetCancelRegistry(adminJobCancelRegistry)
adminJobRunner.Start()
defer adminJobRunner.Stop()
@@ -1632,47 +1896,17 @@ func main() {
provider := auth.NewLocalProvider(userRepo, sessionRepo)
compatDeps.AuthService = auth.NewService(provider, jwtService, sessionRepo, userRepo, nil, nil, nil)
// Access filter resolver for profile-scoped library access.
compatDeps.AccessFilterFn = func(ctx context.Context, userID int, profileID string) catalog.AccessFilter {
if userStoreProvider == nil {
return catalog.AccessFilter{}
}
store, err := userStoreProvider.ForUser(ctx, userID)
if err != nil {
return catalog.AccessFilter{}
}
profile, err := store.GetProfile(ctx, profileID)
if err != nil || profile == nil {
return catalog.AccessFilter{}
}
filter := catalog.AccessFilter{
MaxContentRating: profile.MaxContentRating,
}
if profile.LibraryRestrictionsEnabled && len(profile.AllowedLibraryIDs) > 0 {
filter.AllowedLibraryIDs = profile.AllowedLibraryIDs
}
// Apply user-disabled library IDs.
if raw, err := store.GetSetting(ctx, "disabled_library_ids"); err == nil && raw != "" {
var disabled []int
if json.Unmarshal([]byte(raw), &disabled) == nil && len(disabled) > 0 {
if filter.AllowedLibraryIDs != nil {
filtered := make([]int, 0, len(filter.AllowedLibraryIDs))
disSet := make(map[int]struct{}, len(disabled))
for _, id := range disabled {
disSet[id] = struct{}{}
}
for _, id := range filter.AllowedLibraryIDs {
if _, ok := disSet[id]; !ok {
filtered = append(filtered, id)
}
}
filter.AllowedLibraryIDs = filtered
} else {
filter.DisabledLibraryIDs = disabled
}
}
}
return filter
// Access filter resolver for viewer-scoped library access.
// Backed by the shared access.Resolver so account-level library
// restrictions (users.library_ids), profile restrictions,
// user-disabled libraries, and rating/quality ceilings apply to
// the compat API exactly as they do to the native API.
if userStoreProvider != nil {
compatDeps.AccessFilterFn = jellycompat.NewScopeAccessFilter(access.NewResolver(
userRepo,
userStoreProvider,
nil, // profile tokens unused: compat login already verifies PINs
))
}
}
@@ -1685,6 +1919,28 @@ func main() {
compatSrv.IdleTimeout = 120 * time.Second
}
// ABS-compat listener — dedicated http.Server bound to its own port
// (default :13378) that hosts the Audiobookshelf-compatible API.
// Mirrors the Jellyfin compat layout above. The ABS handler mounts
// onto a fresh chi router here so /ping, /healthcheck, /status, /login,
// /socket.io, etc. own the URL space at the root — no SPA fallback,
// no collision with silo's /api/v1.
var absSrv *http.Server
if (mode == "integrated" || mode == "api") && deps.ABSHandler != nil && cfg.AudiobookshelfCompat.Listen != "" {
absRouter := chi.NewRouter()
absRouter.Use(chimiddleware.Recoverer)
absRouter.Use(chimiddleware.Compress(5))
deps.ABSHandler.Mount(absRouter)
absSrv = &http.Server{
Addr: cfg.AudiobookshelfCompat.Listen,
Handler: absRouter,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 60 * time.Second,
WriteTimeout: 0,
IdleTimeout: 120 * time.Second,
}
}
// Run non-critical startup work in the background so it doesn't delay the
// HTTP listener from accepting connections. Steps run sequentially and stop
// early if the app context is cancelled (shutdown).
@@ -1709,7 +1965,7 @@ func main() {
}()
}
errCh := make(chan error, 2)
errCh := make(chan error, 3)
go func() {
slog.Info("HTTP server listening", "addr", cfg.Server.Listen)
if listenErr := srv.ListenAndServe(); listenErr != nil && listenErr != http.ErrServerClosed {
@@ -1724,6 +1980,14 @@ func main() {
}
}()
}
if absSrv != nil {
go func() {
slog.Info("ABS compat server listening", "addr", absSrv.Addr)
if listenErr := absSrv.ListenAndServe(); listenErr != nil && listenErr != http.ErrServerClosed {
errCh <- fmt.Errorf("abs compat server error: %w", listenErr)
}
}()
}
// Step 11: Wait for termination signal.
sigCh := make(chan os.Signal, 1)
@@ -1734,6 +1998,9 @@ func main() {
case sig := <-sigCh:
appCancel()
slog.Info("received signal, shutting down", "signal", sig)
case <-restartReqCh:
appCancel()
slog.Info("server restart requested, shutting down")
case serverErr := <-errCh:
appCancel()
slog.Error("server error, shutting down", "error", serverErr)
@@ -1753,6 +2020,11 @@ func main() {
slog.Error("jellyfin compat shutdown error", "error", shutdownErr)
}
}
if absSrv != nil {
if shutdownErr := absSrv.Shutdown(shutdownCtx); shutdownErr != nil {
slog.Error("abs compat shutdown error", "error", shutdownErr)
}
}
// 2. Clean up stale sessions.
if sessionCleaner != nil {
@@ -1898,6 +2170,198 @@ func configureS3Clients(cfg *config.Config, deps *api.Dependencies) {
}
}
type markerPluginCapabilityStore interface {
ListEnabled(ctx context.Context) ([]*plugins.Installation, error)
ListCapabilities(ctx context.Context, installationID int) ([]*plugins.Capability, error)
}
type markerPluginRuntimeConfigStore interface {
ListGlobalConfigs(ctx context.Context, installationID int) ([]*plugins.RuntimeConfig, error)
PutGlobalConfig(ctx context.Context, installationID int, key string, value map[string]any) error
}
type markerLegacySettingsStore interface {
Get(ctx context.Context, key string) (string, error)
}
func reloadMarkerPluginProviders(
ctx context.Context,
registry *markers.Registry,
configStore *markers.ProviderConfigStore,
store markerPluginCapabilityStore,
runtimeConfigs markerPluginRuntimeConfigStore,
legacySettings markerLegacySettingsStore,
resolver *markers.PluginResolverAdapter,
) error {
if registry == nil {
return nil
}
var providers []markers.Provider
if store == nil || resolver == nil {
return registry.SetProviders(providers)
}
installations, err := store.ListEnabled(ctx)
if err != nil {
return fmt.Errorf("list enabled plugin installations: %w", err)
}
sort.Slice(installations, func(i, j int) bool {
if installations[i] == nil {
return false
}
if installations[j] == nil {
return true
}
return installations[i].ID < installations[j].ID
})
nextPriority := 1000
for _, installation := range installations {
if installation == nil {
continue
}
capabilities, err := store.ListCapabilities(ctx, installation.ID)
if err != nil {
return fmt.Errorf("list marker provider capabilities for installation %d: %w", installation.ID, err)
}
sort.Slice(capabilities, func(i, j int) bool {
if capabilities[i] == nil {
return false
}
if capabilities[j] == nil {
return true
}
return capabilities[i].ID < capabilities[j].ID
})
for _, capability := range capabilities {
if capability == nil || capability.Type != sdkcapability.MarkerProvider {
continue
}
descriptor, err := plugins.DecodeCapability(capability)
if err != nil {
return fmt.Errorf("decode marker provider capability %d/%s: %w", installation.ID, capability.ID, err)
}
metadataMap := markerCapabilityMetadata(descriptor)
provider, err := markers.NewPluginProvider(markers.PluginProviderOptions{
InstallationID: installation.ID,
CapabilityID: capability.ID,
DisplayName: firstNonEmptyMarkerText(descriptor.GetDisplayName(), capability.ID),
PluginID: installation.PluginID,
RequiredExternalIDs: markers.PluginRequiredExternalIDsFromMetadata(metadataMap),
}, resolver)
if err != nil {
return err
}
providers = append(providers, provider)
priority := nextPriority
nextPriority++
if configuredPriority, ok := markers.PluginDefaultFetchPriorityFromMetadata(metadataMap); ok {
priority = configuredPriority
}
if configStore != nil {
defaultConfig := markers.ProviderConfig{
Provider: provider.ID(),
FetchEnabled: true,
FetchPriority: priority,
ContributeEnabled: false,
ContributeAutoLocal: false,
ContributeMinConfidence: 0.95,
}
if legacy, ok := legacyIntroDBProviderConfig(configStore, installation, capability, provider.ID()); ok {
defaultConfig = legacy
}
if err := configStore.Ensure(ctx, defaultConfig); err != nil {
return err
}
}
if err := copyLegacyIntroDBPluginConfig(ctx, runtimeConfigs, legacySettings, installation, capability); err != nil {
return err
}
}
}
return registry.SetProviders(providers)
}
func legacyIntroDBProviderConfig(
configStore *markers.ProviderConfigStore,
installation *plugins.Installation,
capability *plugins.Capability,
providerID string,
) (markers.ProviderConfig, bool) {
if configStore == nil ||
installation == nil ||
capability == nil ||
installation.PluginID != "silo.theintrodb" ||
capability.ID != "introdb" {
return markers.ProviderConfig{}, false
}
if _, exists := configStore.Get(providerID); exists {
return markers.ProviderConfig{}, false
}
legacy, ok := configStore.Get("introdb")
if !ok {
return markers.ProviderConfig{}, false
}
legacy.Provider = providerID
return legacy, true
}
func copyLegacyIntroDBPluginConfig(
ctx context.Context,
runtimeConfigs markerPluginRuntimeConfigStore,
legacySettings markerLegacySettingsStore,
installation *plugins.Installation,
capability *plugins.Capability,
) error {
if runtimeConfigs == nil ||
legacySettings == nil ||
installation == nil ||
capability == nil ||
installation.PluginID != "silo.theintrodb" ||
capability.ID != "introdb" {
return nil
}
configs, err := runtimeConfigs.ListGlobalConfigs(ctx, installation.ID)
if err != nil {
return fmt.Errorf("list TheIntroDB plugin config: %w", err)
}
for _, config := range configs {
if config != nil && config.Key == "account" {
return nil
}
}
apiKey, err := legacySettings.Get(ctx, "introdb.api_key")
if err != nil {
return fmt.Errorf("load legacy introdb.api_key: %w", err)
}
if strings.TrimSpace(apiKey) == "" {
return nil
}
if err := runtimeConfigs.PutGlobalConfig(ctx, installation.ID, "account", map[string]any{
"api_key": strings.TrimSpace(apiKey),
}); err != nil {
return fmt.Errorf("copy legacy introdb.api_key to plugin config: %w", err)
}
return nil
}
func markerCapabilityMetadata(descriptor *pluginv1.CapabilityDescriptor) map[string]any {
if descriptor == nil || descriptor.GetMetadata() == nil {
return nil
}
return descriptor.GetMetadata().AsMap()
}
func firstNonEmptyMarkerText(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return strings.TrimSpace(value)
}
}
return ""
}
// mapFolderTypeToMediaType maps silo's MediaFolder.Type values
// ("movies", "series", "mixed") to the SDK's MediaType values
// ("movie", "tv", "mixed"). Unknown values map to "mixed".
@@ -1911,3 +2375,14 @@ func mapFolderTypeToMediaType(t string) string {
return "mixed"
}
}
// audiobooksSettingsAdapter bridges catalog.ServerSettingsRepo (which
// exposes Get) to the audiobooks.SettingsReader interface (which
// requires GetString). The two signatures are identical modulo name.
type audiobooksSettingsAdapter struct {
repo *catalog.ServerSettingsRepo
}
func (a *audiobooksSettingsAdapter) GetString(ctx context.Context, key string) (string, error) {
return a.repo.Get(ctx, key)
}
cmd/silo/main_test.go
+15
View File
@@ -132,6 +132,9 @@ func TestBuildLiveSessionSync_UsesTransportPlayMethod(t *testing.T) {
RequestedMediaFileID: 41,
PlayMethod: playback.PlayTranscode,
BasePlayMethod: playback.PlayRemux,
TranscodeHWAccel: "qsv",
Position: 125.5,
IsPaused: true,
},
want: "transcode",
},
@@ -164,6 +167,18 @@ func TestBuildLiveSessionSync_UsesTransportPlayMethod(t *testing.T) {
if got.SessionID != tc.session.ID {
t.Fatalf("SessionID = %q, want %q", got.SessionID, tc.session.ID)
}
if got.ProfileID != tc.session.ProfileID {
t.Fatalf("ProfileID = %q, want %q", got.ProfileID, tc.session.ProfileID)
}
if got.PositionSeconds != tc.session.Position {
t.Fatalf("PositionSeconds = %v, want %v", got.PositionSeconds, tc.session.Position)
}
if got.IsPaused != tc.session.IsPaused {
t.Fatalf("IsPaused = %v, want %v", got.IsPaused, tc.session.IsPaused)
}
if got.TranscodeHWAccel != tc.session.TranscodeHWAccel {
t.Fatalf("TranscodeHWAccel = %q, want %q", got.TranscodeHWAccel, tc.session.TranscodeHWAccel)
}
})
}
}
cmd/silo/marker_plugin_migration_test.go
+85
View File
@@ -0,0 +1,85 @@
package main
import (
"context"
"testing"
"github.com/Silo-Server/silo-server/internal/plugins"
)
type fakeMarkerRuntimeConfigs struct {
configs map[int][]*plugins.RuntimeConfig
puts []plugins.RuntimeConfig
}
func (f *fakeMarkerRuntimeConfigs) ListGlobalConfigs(_ context.Context, installationID int) ([]*plugins.RuntimeConfig, error) {
return append([]*plugins.RuntimeConfig(nil), f.configs[installationID]...), nil
}
func (f *fakeMarkerRuntimeConfigs) PutGlobalConfig(_ context.Context, installationID int, key string, value map[string]any) error {
f.puts = append(f.puts, plugins.RuntimeConfig{InstallationID: installationID, Key: key, Value: value})
f.configs[installationID] = append(f.configs[installationID], &plugins.RuntimeConfig{
InstallationID: installationID,
Key: key,
Value: value,
})
return nil
}
type fakeMarkerLegacySettings map[string]string
func (f fakeMarkerLegacySettings) Get(_ context.Context, key string) (string, error) {
return f[key], nil
}
func TestCopyLegacyIntroDBPluginConfigCopiesAPIKeyOnce(t *testing.T) {
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
installation := &plugins.Installation{ID: 42, PluginID: "silo.theintrodb"}
capability := &plugins.Capability{ID: "introdb"}
if err := copyLegacyIntroDBPluginConfig(
context.Background(),
runtimeConfigs,
fakeMarkerLegacySettings{"introdb.api_key": " legacy-key "},
installation,
capability,
); err != nil {
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
}
if len(runtimeConfigs.puts) != 1 {
t.Fatalf("puts = %d, want 1", len(runtimeConfigs.puts))
}
got := runtimeConfigs.puts[0]
if got.InstallationID != 42 || got.Key != "account" || got.Value["api_key"] != "legacy-key" {
t.Fatalf("put = %+v, want account api_key copy", got)
}
if err := copyLegacyIntroDBPluginConfig(
context.Background(),
runtimeConfigs,
fakeMarkerLegacySettings{"introdb.api_key": "new-key"},
installation,
capability,
); err != nil {
t.Fatalf("second copyLegacyIntroDBPluginConfig: %v", err)
}
if len(runtimeConfigs.puts) != 1 {
t.Fatalf("second copy overwrote config; puts = %d, want 1", len(runtimeConfigs.puts))
}
}
func TestCopyLegacyIntroDBPluginConfigIgnoresOtherPlugins(t *testing.T) {
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
if err := copyLegacyIntroDBPluginConfig(
context.Background(),
runtimeConfigs,
fakeMarkerLegacySettings{"introdb.api_key": "legacy-key"},
&plugins.Installation{ID: 7, PluginID: "silo.other"},
&plugins.Capability{ID: "introdb"},
); err != nil {
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
}
if len(runtimeConfigs.puts) != 0 {
t.Fatalf("puts = %d, want 0 for non-TheIntroDB plugin", len(runtimeConfigs.puts))
}
}
cmd/silo/session_sync.go
+2
View File
@@ -31,8 +31,10 @@ func buildLiveSessionSync(s *playback.Session, reportingNode string) worker.Sess
TargetVideoCodec: s.TargetVideoCodec,
TargetAudioCodec: s.TargetAudioCodec,
TargetBitrateKbps: s.TargetBitrateKbps,
TranscodeHWAccel: s.TranscodeHWAccel,
StartedAt: s.StartedAt,
UpdatedAt: s.UpdatedAt,
PositionSeconds: s.Position,
IsPaused: s.IsPaused,
HasWebSocket: s.HasWebSocket,
}
docker-compose.nvidia.yml
+20
View File
@@ -0,0 +1,20 @@
services:
silo:
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: ${NVIDIA_GPU_COUNT:-1}
capabilities: [gpu]
# Optional distributed-mode example. Most single-host installs should
# just use the integrated service above.
# silo-transcode:
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: ${NVIDIA_GPU_COUNT:-1}
# capabilities: [gpu]
docker-compose.yml
+7 -3
View File
@@ -2,7 +2,7 @@ services:
postgres:
image: pgvector/pgvector:pg18
restart: unless-stopped
shm_size: 1gb
shm_size: ${POSTGRES_SHM_SIZE:-8gb}
environment:
POSTGRES_USER: ${POSTGRES_USER:-silo}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-silo}
@@ -11,8 +11,7 @@ services:
- "5432:5432"
volumes:
- ${SILO_DATA_ROOT:-/opt/silo}/postgres:/var/lib/postgresql
- ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro
command: postgres -c config_file=/etc/postgresql/postgresql.conf
command: ["postgres", "-c", "listen_addresses=*"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-silo}"]
interval: 5s
@@ -40,14 +39,19 @@ services:
DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
REDIS_URL: redis://redis:6379
SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins
POSTGRES_TUNE: ${POSTGRES_TUNE:-auto}
ports:
- "${PORT:-8090}:8080"
- "${JF_PORT:-8096}:8096"
- "${ABS_PORT:-13378}:13378"
volumes:
- ${MEDIA_ROOT:?Set MEDIA_ROOT in .env to the host media path}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro
- ${MEDIA_BOOKS_ROOT:-${MEDIA_ROOT}}:${MEDIA_BOOKS_CONTAINER_ROOT:-${MEDIA_CONTAINER_ROOT:-/mnt/media}/books}:ro
- ${SILO_DATA_ROOT:-/opt/silo}/plugins:/var/lib/silo/plugins
- ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode
- ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro
- ${SILO_DATA_ROOT:-/opt/silo}/audiobook-covers:/var/lib/silo/audiobook-covers
- /proc/meminfo:/host/proc/meminfo:ro
depends_on:
postgres:
condition: service_healthy
docs/superpowers/notes/2026-05-27-abs-wire-shape-verification.md
+138
View File
@@ -0,0 +1,138 @@
# ABS Wire-Shape Verification (post collections-unify cutover)
Breadcrumbs for the next person debugging an ABS endpoint wire-shape issue
after the canonical-tables cutover (migration 156 + commits `0dc830e`,
`8c7fe1b`, `b64ce17`).
The Go in-memory structs (`abs.Collection`, `abs.Playlist`,
`abs.SmartCollection`, `abs.CollectionItem`, `abs.PlaylistItem`) have **no
`json:"..."` struct tags**. The JSON wire contract is defined entirely by
the `*ToABS()` map-builder helpers in `internal/audiobooks/abs/`. As long as
the store layer populates the struct fields with the same values, the wire
shape is preserved. The rewrites in `0dc830e`, `8c7fe1b`, `b64ce17` did NOT
modify the emitters — only the SQL-backed store implementations.
## Envelope tests — which one guards which endpoint
Run with `go test ./internal/audiobooks/abs/ -run Envelope -v -count=1`.
| Test file | Functions | Guards |
|---|---|---|
| `collections_envelope_test.go` | `TestCollectionEnvelope_HasRequiredKeys`, `TestCollectionListShape_OmitsBooks` | `collectionToABS` keys; list-shape (no `books`) vs detail-shape (with `books`) for `GET /api/collections`, `GET /api/collections/{id}`, `GET /api/libraries/{id}/collections`, and all POST/PATCH/DELETE collection endpoints |
| `playlists_envelope_test.go` | `TestPlaylistEnvelope_HasRequiredKeys`, `TestPlaylistEnvelope_OmitsCoverPathWhenEmpty`, `TestPlaylistListShape_OmitsItems` | `playlistToABS` keys; list-shape (no `items`) vs detail-shape (with `items`); `coverPath` is omitted when `CoverItem == ""` — covers `GET /api/playlists`, `GET /api/playlists/{id}`, `GET /api/libraries/{id}/playlists`, batch and item-add/remove endpoints |
| `smart_collections_envelope_test.go` | `TestSmartCollectionEnvelope_HasRequiredKeys`, `TestSmartCollectionEnvelope_EmptyQueryDef` | `smartCollectionToABS` keys; `queryDef` decoded from raw JSONB bytes into nested object, empty bytes → `{}` — covers `GET /api/me/smart-collections`, `GET /api/me/smart-collections/{id}`, POST/PATCH equivalents |
| `bookmarks_envelope_test.go` | `TestBookmarkEnvelope_HasRequiredKeys` | Bookmarks emitter (separate from this cutover, not affected by migration 156) |
| `login_envelope_test.go` | `TestLoginEnvelope_HasRequiredKeys` and three xReturnTokens / displayName variants | Login envelope (not affected by migration 156) |
In addition, handler-level round-trip tests live in
`playlists_handler_test.go` and `bookmarks_handler_test.go`. There is NO
snapshot/goldenfile harness in the repo today — these envelope tests are
the primary regression guard.
## Manual live-DB diff procedure
For a pre/post-deploy wire-shape verification against a live silo, see the
plan's Task 5 "manual verification" section at
`docs/superpowers/plans/2026-05-27-collections-unify-3-abs-adapters.md`
(§ `Task 5: Wire-shape regression test`). Summary:
1. Pre-cutover, seed one of each (collection, playlist with item,
smart collection) via the old `abs_*` tables, then capture each list
endpoint's response to `/tmp/wire_before_<kind>.json` using a curl
against the running silo with a valid ABS bearer token (HS256 JWT —
minted by the login flow, NOT the raw `abs_sessions.token` value).
2. Apply migration 156. Seed equivalent rows in `user_personal_collections`
with the same IDs and content. Capture again to
`/tmp/wire_after_<kind>.json`.
3. `diff /tmp/wire_before_<kind>.json /tmp/wire_after_<kind>.json` for each
`kind in {collections,playlists,smart_collections}`. Expected: empty
diff.
This is an MR-description-level manual step, not a committed test.
## Intentionally-zero fields after the rewrite
These wire keys are still emitted, but the store always populates the
in-memory field with the zero value because the canonical
`user_personal_collections` schema has no analog column (per spec §6 of the
collections-unify plan). They are NOT bugs — do not "fix" them by reaching
for some other column.
| In-memory field | Wire key | Zero value | Spec ref | Disposition |
|---|---|---|---|---|
| `abs.Playlist.CoverItem` | `coverPath` | `""` (key omitted entirely when empty — see `playlistToABS`) | spec §6.1 | Dropped. PATCH `coverPath` body field is silently ignored by the store. Cover regeneration from first-item poster is the chosen long-term path. |
| `abs.SmartCollection.Color` | `color` | `""` (key always emitted as empty string) | spec §6.3 | Deferred. No column on `user_personal_collections`. Wire key stays present for client compatibility. |
| `abs.SmartCollection.IsPinned` | `isPinned` | `false` (key always emitted) | spec §6.2 | Deferred. Same rationale. |
If you're adding a "Pin this smart collection" feature later, the column
needs to land in a new migration on `user_personal_collections` first;
don't try to thread it through some adjacent column.
## Canonical mapping — struct field → source column
The full pre-cutover wire contract was captured in a working note that does
not persist (`/tmp/abs_wire_contract.md`). The essentials are reproduced
here so the next maintainer doesn't have to re-derive them.
All three struct families now read from `user_personal_collections`
(and `user_personal_collection_items` for collections + playlists),
discriminated by `collection_type IN ('manual','playlist','smart')`.
### `abs.Collection` (`collection_type = 'manual'`)
| Field | Source column |
|---|---|
| ID | `user_personal_collections.id` |
| UserID | `user_personal_collections.user_id::text` (column is `integer`) |
| ProfileID | `user_personal_collections.profile_id` |
| Name | `user_personal_collections.name` |
| Description | `user_personal_collections.description` |
| IsPublic | `user_personal_collections.is_shared` |
| CreatedAt | `user_personal_collections.created_at` |
| UpdatedAt | `user_personal_collections.updated_at` |
`abs.CollectionItem` reads `user_personal_collection_items` with
`sub_item_id = ''` filter (the manual-collection sentinel established in
migration 156 step 1). LibraryItemID ← `media_item_id`. ORDER BY
`added_at ASC`.
### `abs.Playlist` (`collection_type = 'playlist'`)
Same column mapping as `abs.Collection` (modulo `collection_type` filter)
EXCEPT `CoverItem` which is always `""` — see "Intentionally-zero fields"
above.
`abs.PlaylistItem` reads `user_personal_collection_items` with NO
`sub_item_id` filter (playlists can carry episode entries). Mapping:
LibraryItemID ← `media_item_id`, EpisodeID ← `sub_item_id`,
Position ← `position`. ORDER BY `position ASC, added_at ASC`.
### `abs.SmartCollection` (`collection_type = 'smart'`)
Same column mapping as `abs.Collection` EXCEPT:
- `Color`, `IsPinned` → always zero (see above).
- `QueryDef` ← `user_personal_collections.query_definition` (JSONB → `[]byte`
round-trip; column is `NOT NULL DEFAULT '{}'::jsonb` per migration 016).
No items table — smart-collection membership is evaluated at request time
via the `smartcoll` package.
### Wire-shape quirks worth remembering
- Collection/Playlist emit `lastUpdate` (NOT `updatedAt`). SmartCollection
emits `updatedAt`. Cross-struct inconsistency, carry forward verbatim.
- All timestamps are `UnixMilli()` int64, NOT RFC3339 strings.
- `ProfileID` is carried in memory but NEVER emitted on the wire — it's
scope/auth only.
- The list vs detail shape distinction is implicit: list responses pass
`nil` for the items/books slice; the emitter then omits the key
entirely. Clients differentiate on key presence.
## Verification status (2026-05-27)
- All 13 envelope tests pass (run: `go test ./internal/audiobooks/abs/
-run Envelope -v -count=1`).
- Full audiobooks test suite passes (`go test ./internal/audiobooks/...
-short -count=1 -timeout 120s`).
- Live-DB diff was NOT executed in CI — see manual procedure above.
docs/superpowers/plans/2026-05-24-audiobook-ui-redesign.md
+3072
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-24-audiobooks-absorption-1-discovery-schema.md
+916
View File
@@ -0,0 +1,916 @@
# Audiobooks Absorption — Sub-plan 1: Discovery + Schema
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Resolve open data-model questions from the audiobooks spec, then land the foundational schema (two new tables, optional one column add, one feature-flag setting) and a compiling-but-empty `internal/audiobooks/` package scaffold. No user-visible behavior change after this sub-plan; silo continues to operate exactly as it does today.
**Architecture:** Six tasks total. Task 1 audits silo-server's existing schema/code and writes a findings doc that locks the remaining decisions. Tasks 2–5 land additive SQL migrations (idempotent up + symmetric down). Task 6 creates a Go package skeleton with one no-op service constructor wired into the dependency-injection point in `cmd/silo`, which proves the new package compiles inside the rest of the binary without changing any runtime behavior.
**Tech Stack:** Go 1.26 + pgx; PostgreSQL 18 (pgvector image); plain `.up.sql` / `.down.sql` migrations applied at startup by `internal/database/migrate.go`. Frontend out of scope for this sub-plan.
**Source spec:** `docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md`
---
## File Structure
| Path | Created/Modified | Purpose |
|---|---|---|
| `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` | Create | Findings doc produced by Task 1. Locks the remaining schema/code decisions. |
| `migrations/147_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
| `migrations/147_abs_sessions.down.sql` | Create | Drop the table. |
| `migrations/157_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
| `migrations/157_podcast_feeds.down.sql` | Create | Drop the table. |
| `migrations/159_media_folders_kind_noop.up.sql` | Create (conditional — see Task 4 step 1) | Document `media_folders.type='audiobooks'`; no column change needed. |
| `migrations/159_media_folders_kind_noop.down.sql` | Create (same condition) | No-op rollback. |
| `migrations/160_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
| `migrations/160_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
| `internal/audiobooks/doc.go` | Create | Package comment + intent. |
| `internal/audiobooks/service.go` | Create | Empty `Service` struct, `New` constructor, `Enabled()` reader. |
| `internal/audiobooks/service_test.go` | Create | One unit test proving the package compiles and `Enabled()` returns the setting value. |
| `cmd/silo/main.go` | Modify | Construct `audiobooks.New(...)` once during startup so the scaffold is referenced. No routes are mounted, no tasks registered. |
Migrations and the scaffold compile and run independently from the audiobooks ports that arrive in later sub-plans.
---
## Task 1: Discovery Audit
Audits silo-server's existing schema and code to resolve the spec's open
Risk questions. Produces one findings document and one commit. No code
changes elsewhere.
**Files:**
- Create: `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md`
### Step 1.1: Create the findings doc skeleton
- [ ] **Create** `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` with this exact content:
```markdown
# Audiobooks Absorption — Discovery Findings
Produced by sub-plan 1, Task 1. Locks data-model and integration
decisions for migrations 139–142 and downstream sub-plans.
## D1 — Next migration number
(Filled in step 1.2)
## D2 — `media_libraries` kind/type column
(Filled in step 1.3)
## D3 — `media_files.chapters` JSONB shape
(Filled in step 1.4)
## D4 — `user_watch_progress` scoping (profile vs user)
(Filled in step 1.5)
## D5 — `user_playback_sessions` audiobook fit
(Filled in step 1.6)
## D6 — `people` / `item_people` role conventions
(Filled in step 1.7)
## D7 — Catalog FTS handling of `type='audiobook'`
(Filled in step 1.8)
## D8 — First-party scheduled-task registration
(Filled in step 1.9)
```
### Step 1.2: D1 — Migration number
- [ ] **Run:** `ls migrations/ | grep -E '^[0-9]+_' | sort -t_ -k1 -n | tail -3`
Expected output is three filenames whose numeric prefixes are consecutive.
Whatever the highest prefix is, the next available number is **(highest + 1)**.
- [ ] **Append to the findings doc** under the D1 heading:
```text
Highest existing migration: <NN>_<name>.up.sql
Next available number: <NN+1>
Sub-plan 1 will use migrations <NN+1> through <NN+4>.
```
(Replace the placeholders with the observed values.) This locks the
numbering used by Tasks 2–5. If the next number is not 139, update the
filenames in Tasks 2–5 accordingly when you reach them.
### Step 1.3: D2 — `media_libraries` kind/type column
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.media_libraries"`
Inspect the column list. Look for any column whose name or comment
suggests a per-library *kind* / *type* / *category* discriminator
(candidate names: `kind`, `library_type`, `category`, `media_type`,
`content_type`).
- [ ] **Append to the findings doc** under the D2 heading exactly one of:
- `No existing kind/type column on media_libraries. Task 4 adds 'kind' as planned.`
- `Existing column '<NAME>' (<TYPE>) discriminates library content. Task 4 becomes a no-op migration; the audiobook scanner branch will read '<NAME>' instead of 'kind'.`
### Step 1.4: D3 — `media_files.chapters` JSONB shape
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT chapters FROM media_files WHERE chapters IS NOT NULL AND jsonb_array_length(chapters) > 0 LIMIT 1;"`
This returns one sample chapter array used by silo's existing video
chapter feature (added in migration 066).
- [ ] **Append to the findings doc** under the D3 heading:
```text
Sample chapter JSON (live data):
<paste the JSON exactly as returned, no edits>
Sub-plan 2 (scanner) MUST emit objects with the same keys when writing
audiobook chapters so the existing player and serialization code accept
them without changes.
```
### Step 1.5: D4 — `user_watch_progress` scoping
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.user_watch_progress"`
- [ ] **Append to the findings doc** under the D4 heading exactly one of:
- `user_watch_progress is profile-scoped: column '<profile_id>' (FK to user_profiles). Audiobook progress slots in directly.`
- `user_watch_progress is user-scoped only (no profile column). Audiobook progress will share user state across all household profiles unless Sub-plan 3 adds a profile_id column. RAISE THIS AS A RISK in sub-plan 2 brainstorming.`
### Step 1.6: D5 — `user_playback_sessions` fit
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.user_playback_sessions"`
- [ ] **Append to the findings doc** under the D5 heading:
```text
Column list:
<paste the \d output verbatim>
Columns required by audiobook sessions: media_item_id (or equivalent),
profile/user FK, started_at, current_position_seconds (or equivalent),
status. Mark any required column as MISSING and surface in sub-plan 3.
```
### Step 1.7: D6 — `people` / `item_people` role conventions
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.item_people"
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT DISTINCT role FROM item_people ORDER BY role;"
```
- [ ] **Append to the findings doc** under the D6 heading:
```text
item_people.role storage: <TEXT | enum named '<name>' | CHECK constraint>
Existing role values in use: <comma-separated list from the SELECT>
Sub-plan 2 will UPSERT 'author' and 'narrator' into item_people for
audiobook items. If the role column is constrained by an enum or CHECK,
Sub-plan 2 must add 'author' and 'narrator' to the constraint as part of
its first migration.
```
### Step 1.8: D7 — FTS handling of new `type` values
- [ ] **Run:**
```bash
grep -lE "type\s*=\s*'(movie|series|episode)'" migrations/*.up.sql | head -5
```
These migrations build/maintain silo's title search. Open each one and
scan for whether the type filter is hardcoded (`type IN ('movie','series')`)
or generic (no type filter, indexes everything).
- [ ] **Append to the findings doc** under the D7 heading:
```text
Indexes / generated columns that filter by media_items.type:
<bulleted list of (migration_file, what it filters)>
Verdict: audiobooks WILL / WILL NOT be FTS-searchable out of the box.
If WILL NOT: Sub-plan 3 must include an extra migration that extends the
type filter to include 'audiobook' and 'podcast'.
```
### Step 1.9: D8 — Scheduled-task registration
- [ ] **Run:**
```bash
grep -rEn "RegisterTask|TaskManager|RegisterScheduled|task\.Register" cmd/silo/main.go internal --include='*.go' | head -10
```
- [ ] **Append to the findings doc** under the D8 heading:
```text
First-party scheduled tasks register at: <file:line>
Registration call shape: <one-line copy of the call as used by an
existing first-party task such as smart-count refresh>
Sub-plan 5 (podcasts) will register podcastfeed.Refresher at the same
call site using the same pattern.
```
### Step 1.10: Commit
- [ ] **Run:**
```bash
git add docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md
git commit -m "$(cat <<'EOF'
docs(audiobooks): discovery findings for absorption sub-plan 1
Locks schema/code decisions for migrations 139-142 and downstream
sub-plans. Resolves open Risk questions from the absorption design spec.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
EOF
)"
```
---
## Task 2: Migration 147 — `abs_sessions`
Parallel of `jellycompat_sessions` for Audiobookshelf clients. Lets ABS
mobile/desktop apps reconnect without re-authenticating against silo's
main `auth_sessions`.
**Files:**
- Create: `migrations/147_abs_sessions.up.sql`
- Create: `migrations/147_abs_sessions.down.sql`
> **Note:** If Task 1, Step 1.2 found the next number is not 139, rename
> both filenames to match. All other tasks downstream of 139 shift by the
> same delta.
### Step 2.1: Write the failing assertion
- [ ] **Run** (this should error — the table does not exist yet):
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
```
Expected: `Did not find any relation named "public.abs_sessions".`
### Step 2.2: Create the up migration
- [ ] **Create** `migrations/147_abs_sessions.up.sql` with:
```sql
-- Audiobookshelf-compatible client sessions. Parallel to
-- jellycompat_sessions: each row identifies an ABS mobile/desktop
-- client by its device + token so it can reconnect without
-- re-authenticating against silo's main auth_sessions.
CREATE TABLE IF NOT EXISTS public.abs_sessions (
id BIGSERIAL PRIMARY KEY,
user_id INTEGER NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
token_hash TEXT NOT NULL,
device_id TEXT NOT NULL,
device_name TEXT,
client_name TEXT,
client_version TEXT,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
last_seen_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
revoked_at TIMESTAMP WITH TIME ZONE
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token_hash
ON public.abs_sessions (token_hash);
CREATE INDEX IF NOT EXISTS idx_abs_sessions_user_device
ON public.abs_sessions (user_id, device_id);
CREATE INDEX IF NOT EXISTS idx_abs_sessions_last_seen
ON public.abs_sessions (last_seen_at);
```
### Step 2.3: Create the down migration
- [ ] **Create** `migrations/147_abs_sessions.down.sql` with:
```sql
DROP INDEX IF EXISTS public.idx_abs_sessions_last_seen;
DROP INDEX IF EXISTS public.idx_abs_sessions_user_device;
DROP INDEX IF EXISTS public.idx_abs_sessions_token_hash;
DROP TABLE IF EXISTS public.abs_sessions;
```
### Step 2.4: Apply the migration by restarting silo
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
- [ ] **Wait until healthy:**
```bash
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
```
- [ ] **Verify silo applied the migration:**
```bash
sudo docker logs --since 2m silo-prod-silo-1 2>&1 | grep -i 'database migrations applied'
```
Expected: one line containing `database migrations applied`.
### Step 2.5: Verify the table exists and is usable
- [ ] **Run** (each command should now succeed):
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 147;"
```
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=147`.
- [ ] **Run** a sample insert/delete to prove constraints work:
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
INSERT INTO abs_sessions (user_id, token_hash, device_id, device_name, client_name, client_version)
VALUES ((SELECT id FROM users ORDER BY id LIMIT 1), 'test-token-hash-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
DELETE FROM abs_sessions WHERE token_hash = 'test-token-hash-001';
"
```
Expected: `INSERT 0 1` then `DELETE 1`. (At least one user must exist.)
### Step 2.6: Commit
- [ ] **Run:**
```bash
git add migrations/147_abs_sessions.up.sql migrations/147_abs_sessions.down.sql
git commit -m "$(cat <<'EOF'
feat(audiobooks): migration 147 add abs_sessions table
Parallel of jellycompat_sessions for Audiobookshelf-compatible clients.
Lets ABS mobile/desktop apps maintain a device-bound session that
silo's audiobooks/abs handlers will validate.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
EOF
)"
```
---
## Task 3: Migration 157 — `podcast_feeds`
One row per subscribed podcast. The podcast itself lives in `media_items`
(`type='podcast'`); this side table carries the RSS-specific metadata the
feed refresher needs.
**Files:**
- Create: `migrations/157_podcast_feeds.up.sql`
- Create: `migrations/157_podcast_feeds.down.sql`
### Step 3.1: Write the failing assertion
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
```
Expected: `Did not find any relation named "public.podcast_feeds".`
### Step 3.2: Create the up migration
- [ ] **Create** `migrations/157_podcast_feeds.up.sql` with:
```sql
-- RSS-feed metadata for subscribed podcasts. One row per podcast
-- media_items row; the feed refresher polls feed_url every
-- refresh_interval_seconds and upserts new episodes into the existing
-- episodes table.
CREATE TABLE IF NOT EXISTS public.podcast_feeds (
media_item_id TEXT PRIMARY KEY
REFERENCES public.media_items(content_id) ON DELETE CASCADE,
feed_url TEXT NOT NULL,
etag TEXT,
last_modified TEXT,
last_refreshed_at TIMESTAMP WITH TIME ZONE,
last_refresh_error TEXT,
refresh_interval_seconds INTEGER NOT NULL DEFAULT 600,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_podcast_feeds_feed_url
ON public.podcast_feeds (feed_url);
CREATE INDEX IF NOT EXISTS idx_podcast_feeds_due_for_refresh
ON public.podcast_feeds (last_refreshed_at);
```
> **Note:** `media_items.content_id` is TEXT (silo uses snowflake-style
> string IDs, not BIGSERIAL — confirmed by the FK targets in other
> migrations). If Task 1 Step 1.4 sampled a numeric `content_id`,
> change the column type to match. Otherwise leave as TEXT.
### Step 3.3: Create the down migration
- [ ] **Create** `migrations/157_podcast_feeds.down.sql` with:
```sql
DROP INDEX IF EXISTS public.idx_podcast_feeds_due_for_refresh;
DROP INDEX IF EXISTS public.idx_podcast_feeds_feed_url;
DROP TABLE IF EXISTS public.podcast_feeds;
```
### Step 3.4: Apply the migration
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
- [ ] **Wait until healthy:**
```bash
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
```
### Step 3.5: Verify
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 157;"
```
Expected: column list printed; one row with `version=157`.
### Step 3.6: Commit
- [ ] **Run:**
```bash
git add migrations/157_podcast_feeds.up.sql migrations/157_podcast_feeds.down.sql
git commit -m "$(cat <<'EOF'
feat(audiobooks): migration 157 add podcast_feeds table
Side table on media_items for RSS-subscribed podcasts. Holds feed URL,
ETag/Last-Modified for conditional fetches, last-refresh timestamp, and
the per-feed refresh interval consumed by the upcoming
podcastfeed.Refresher scheduled task.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
EOF
)"
```
---
## Task 4: Migration 159 — `media_libraries.kind` audiobook value (no-op)
Adds a discriminator column so the scanner can branch on
`audiobooks`/`podcasts` libraries without filename heuristics.
> **CONDITIONAL:** If Task 1 Step 1.3 (D2) recorded *"Existing column
> '<NAME>' discriminates library content"*, swap this whole task for a
> documented no-op: create both migration files as no-op SQL (`-- intentionally empty: existing column '<NAME>' covers this need`)
> and adjust downstream sub-plans to read `'<NAME>'` instead of `kind`.
> Otherwise, proceed with the steps below.
**Files:**
- Create: `migrations/159_media_folders_kind_noop.up.sql`
- Create: `migrations/159_media_folders_kind_noop.down.sql`
### Step 4.1: Write the failing assertion
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT kind FROM media_libraries LIMIT 1;"
```
Expected: `column "kind" does not exist`.
### Step 4.2: Create the up migration
- [ ] **Create** `migrations/159_media_folders_kind_noop.up.sql` with:
```sql
-- Per-library content discriminator. Lets the scanner choose the right
-- parser (movie/tv extensions vs audiobook/podcast extensions) without
-- filename heuristics. Existing rows default to 'movies' to preserve
-- current behavior; operators set 'audiobooks' or 'podcasts' on the
-- libraries they want the new flavor for.
ALTER TABLE public.media_libraries
ADD COLUMN IF NOT EXISTS kind TEXT NOT NULL DEFAULT 'movies';
ALTER TABLE public.media_libraries
ADD CONSTRAINT media_libraries_kind_check
CHECK (kind IN ('movies', 'tv', 'audiobooks', 'podcasts'));
CREATE INDEX IF NOT EXISTS idx_media_libraries_kind
ON public.media_libraries (kind);
```
### Step 4.3: Create the down migration
- [ ] **Create** `migrations/159_media_folders_kind_noop.down.sql` with:
```sql
DROP INDEX IF EXISTS public.idx_media_libraries_kind;
ALTER TABLE public.media_libraries
DROP CONSTRAINT IF EXISTS media_libraries_kind_check;
ALTER TABLE public.media_libraries
DROP COLUMN IF EXISTS kind;
```
### Step 4.4: Apply the migration
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
- [ ] **Wait until healthy** (same command as Task 2 Step 2.4).
### Step 4.5: Verify
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
SELECT id, name, kind FROM media_libraries ORDER BY id;
"
```
Expected: every existing library row has `kind='movies'`.
- [ ] **Run** to confirm the CHECK rejects bad values:
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
INSERT INTO media_libraries (id, name, kind) VALUES (-9999, 'bad-test', 'garbage');
"
```
Expected: `ERROR: new row for relation "media_libraries" violates check constraint "media_libraries_kind_check"`.
- [ ] **Run** to clean up any test row that snuck through (no-op if the previous step rejected as expected):
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "DELETE FROM media_libraries WHERE id = -9999;"
```
### Step 4.6: Commit
- [ ] **Run:**
```bash
git add migrations/159_media_folders_kind_noop.up.sql migrations/159_media_folders_kind_noop.down.sql
git commit -m "$(cat <<'EOF'
feat(audiobooks): migration 159 document audiobooks media folder type
Per-library discriminator that lets the scanner pick the right parser
('movies', 'tv', 'audiobooks', 'podcasts') instead of inferring from
filename. Existing libraries default to 'movies'; operators set the new
values on the libraries they want indexed with audiobook/podcast logic.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
EOF
)"
```
---
## Task 5: Migration 160 — `audiobooks.enabled` feature flag
Adds a `server_settings` row so subsequent sub-plans can branch on
"audiobooks compiled in but turned off" vs "fully live". Default is
`false` so this sub-plan's landing is a strict no-op for users.
**Files:**
- Create: `migrations/160_audiobooks_feature_flag.up.sql`
- Create: `migrations/160_audiobooks_feature_flag.down.sql`
### Step 5.1: Write the failing assertion
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
SELECT value FROM server_settings WHERE key = 'audiobooks.enabled';
"
```
Expected: zero rows.
### Step 5.2: Create the up migration
- [ ] **Create** `migrations/160_audiobooks_feature_flag.up.sql` with:
```sql
-- Master kill-switch for the absorbed audiobooks feature. Defaults to
-- 'false' so landing migrations 147/157/159/160 is a no-op for users; operators
-- flip this to 'true' at cutover.
INSERT INTO server_settings (key, value) VALUES ('audiobooks.enabled', 'false')
ON CONFLICT (key) DO NOTHING;
```
### Step 5.3: Create the down migration
- [ ] **Create** `migrations/160_audiobooks_feature_flag.down.sql` with:
```sql
DELETE FROM server_settings WHERE key = 'audiobooks.enabled';
```
### Step 5.4: Apply and verify
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
- [ ] **Wait until healthy** (same command).
- [ ] **Run:**
```bash
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
SELECT key, value FROM server_settings WHERE key = 'audiobooks.enabled';
"
```
Expected: one row, `value='false'`.
### Step 5.5: Commit
- [ ] **Run:**
```bash
git add migrations/160_audiobooks_feature_flag.up.sql migrations/160_audiobooks_feature_flag.down.sql
git commit -m "$(cat <<'EOF'
feat(audiobooks): migration 160 add audiobooks.enabled flag
Server-settings row that gates the absorbed audiobooks feature.
Defaults to 'false' so sub-plan 1 lands as a strict no-op; subsequent
sub-plans branch on this flag and operators flip it to 'true' at
cutover.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
EOF
)"
```
---
## Task 6: Scaffold `internal/audiobooks/` package
Creates an empty but compiling Go package, wires its constructor into
`cmd/silo/main.go` so the package is referenced from the binary, and
ships a unit test that proves the `Enabled()` reader returns the flag
value seeded by migration 160.
This task proves the package compiles inside the rest of silo without
changing any runtime behavior. No routes are mounted. No scheduled
tasks are registered. No DB writes happen.
**Files:**
- Create: `internal/audiobooks/doc.go`
- Create: `internal/audiobooks/service.go`
- Create: `internal/audiobooks/service_test.go`
- Modify: `cmd/silo/main.go` (one new constructor call near the other service constructions)
### Step 6.1: Write the failing test
- [ ] **Create** `internal/audiobooks/service_test.go` with:
```go
package audiobooks
import (
"context"
"errors"
"testing"
)
type fakeSettingsReader struct {
value string
err error
}
func (f *fakeSettingsReader) GetString(_ context.Context, key string) (string, error) {
if key != "audiobooks.enabled" {
return "", errors.New("unexpected key: " + key)
}
return f.value, f.err
}
func TestServiceEnabledReadsFlag(t *testing.T) {
cases := []struct {
name string
stored string
want bool
}{
{"flag true", "true", true},
{"flag false", "false", false},
{"flag empty defaults false", "", false},
{"flag garbage defaults false", "yes-please", false},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
svc := New(&fakeSettingsReader{value: tc.stored})
got, err := svc.Enabled(context.Background())
if err != nil {
t.Fatalf("Enabled returned error: %v", err)
}
if got != tc.want {
t.Fatalf("Enabled = %v, want %v", got, tc.want)
}
})
}
}
func TestServiceEnabledPropagatesError(t *testing.T) {
wantErr := errors.New("db down")
svc := New(&fakeSettingsReader{err: wantErr})
_, err := svc.Enabled(context.Background())
if !errors.Is(err, wantErr) {
t.Fatalf("Enabled error = %v, want %v wrapped", err, wantErr)
}
}
```
### Step 6.2: Run the test to verify it fails
- [ ] **Run:** `go test ./internal/audiobooks/...`
Expected: build failure with messages like `undefined: New` and
`package audiobooks not found`.
### Step 6.3: Write the package doc
- [ ] **Create** `internal/audiobooks/doc.go` with:
```go
// Package audiobooks owns silo's first-party audiobook + podcast feature,
// absorbed from the historical silo-plugin-audiobooks. Sub-plan 1 lands
// only the package scaffold and the kill-switch reader; ABS-compat REST,
// Socket.io, scanner branches, podcast feed refresh, and the silo SPA
// pages arrive in later sub-plans.
//
// See docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md
// for the design and docs/superpowers/plans/2026-05-24-audiobooks-*.md
// for the staged implementation plans.
package audiobooks
```
### Step 6.4: Write the service
- [ ] **Create** `internal/audiobooks/service.go` with:
```go
package audiobooks
import (
"context"
"fmt"
)
// SettingsReader is the minimal slice of the server-settings store that
// the audiobooks service needs. The production implementation is
// internal/serversettings.Store (or whatever silo names that helper at
// wiring time); tests pass a fake.
type SettingsReader interface {
GetString(ctx context.Context, key string) (string, error)
}
// Service is the audiobooks feature's top-level orchestrator. Sub-plan 1
// exposes only Enabled(); subsequent sub-plans hang additional methods
// off Service as new capabilities (scanner branches, ABS handlers, etc.)
// come online.
type Service struct {
settings SettingsReader
}
// New constructs a Service. The constructor takes the dependencies it
// will actually use; current sub-plan needs only the settings reader.
func New(settings SettingsReader) *Service {
return &Service{settings: settings}
}
// Enabled reports whether the audiobooks feature flag (set by
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
// Any value other than the literal string "true" reads as false; this matches how silo
// treats other boolean server_settings rows.
func (s *Service) Enabled(ctx context.Context) (bool, error) {
if s == nil || s.settings == nil {
return false, nil
}
value, err := s.settings.GetString(ctx, "audiobooks.enabled")
if err != nil {
return false, fmt.Errorf("audiobooks: read audiobooks.enabled: %w", err)
}
return value == "true", nil
}
```
### Step 6.5: Run the test to verify it passes
- [ ] **Run:** `go test ./internal/audiobooks/...`
Expected: `ok github.com/Silo-Server/silo-server/internal/audiobooks ...`
### Step 6.6: Wire the constructor into `cmd/silo/main.go`
The audit in Task 1 Step 1.9 located the function where first-party
services are constructed at startup. Open `cmd/silo/main.go`, find the
block where other internal services (e.g. metadata, downloads, catalog)
are constructed and pass a settings store to.
- [ ] **Run** to find the right insertion point:
```bash
grep -nE 'serversettings|server_settings|settingsStore|metadata\.NewService|downloads\.NewService' cmd/silo/main.go | head -10
```
Note the file:line of the existing service construction block.
- [ ] **Modify** `cmd/silo/main.go`: in the service-construction block (immediately after another similar `service := pkg.New(...)` line), add:
```go
audiobooksService := audiobooks.New(settingsStore)
_ = audiobooksService // referenced by sub-plan 2 onward; no behavior in sub-plan 1
```
Where `settingsStore` is whatever the existing services pass in for
the server-settings reader. The `_ = audiobooksService` line keeps Go
from rejecting an unused variable; it gets removed in sub-plan 2 when
the variable is actually used.
- [ ] **Add** the import to the import block at the top of `cmd/silo/main.go`:
```go
"github.com/Silo-Server/silo-server/internal/audiobooks"
```
### Step 6.7: Verify everything still compiles and tests pass
- [ ] **Run:** `go build ./...`
Expected: exit 0, no output.
- [ ] **Run:** `go test ./internal/audiobooks/... ./cmd/...`
Expected: all passes.
- [ ] **Run:** `make lint`
Expected: no new lint findings introduced by this task.
### Step 6.8: Smoke-test the running binary
- [ ] **Run:** `sudo docker compose -p silo-prod up -d --force-recreate silo`
- [ ] **Wait until healthy** (same command as Task 2 Step 2.4).
- [ ] **Run:**
```bash
curl -sS -o /dev/null -w "HTTP %{http_code}\n" http://localhost:8090/api/v1/health
curl -sS -o /dev/null -w "HTTP %{http_code}\n" http://localhost:8090/
```
Expected: same status codes silo returned before this sub-plan
started (typically `200` for `/`, `404` for `/api/v1/health` since
that path was 404 in the baseline logs). No new errors in `docker
logs silo-prod-silo-1`.
### Step 6.9: Commit
- [ ] **Run:**
```bash
git add internal/audiobooks/doc.go internal/audiobooks/service.go internal/audiobooks/service_test.go cmd/silo/main.go
git commit -m "$(cat <<'EOF'
feat(audiobooks): scaffold internal/audiobooks package
Empty-but-compiling Service that reads the audiobooks.enabled feature
flag from server_settings. Wired into cmd/silo so the package is
referenced from the binary; no routes mounted, no scheduled tasks
registered, no DB writes. Subsequent sub-plans hang scanner branches,
ABS handlers, Socket.io, podcast refresher, and SPA pages off this
Service.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
EOF
)"
```
---
## Self-review (done by the planner before handoff)
**Spec coverage:**
| Spec section | Task(s) that cover it |
|---|---|
| Data model: reused tables | Task 1 (verification only) |
| Data model: `abs_sessions` (migration 147) | Task 2 |
| Data model: `podcast_feeds` (migration 157) | Task 3 |
| Data model: `media_folders.type` audiobook value (migration 159 no-op) | Task 4 |
| Data model: data migration = none | n/a — confirmed in spec |
| Feature flag (`audiobooks.enabled`) | Task 5 |
| Architecture: `internal/audiobooks/` package | Task 6 |
| Risks: `media_libraries` column check | Task 1, Step 1.3 |
| Risks: `media_files.chapters` shape | Task 1, Step 1.4 |
| Risks: progress profile-scoping | Task 1, Step 1.5 |
| Risks: FTS handling of `audiobook` | Task 1, Step 1.8 |
| Risks: `people` role conventions | Task 1, Step 1.7 |
Out of sub-plan 1 scope (deferred to later sub-plans):
ABS REST port, Socket.io port, scanner branches, podcast refresher,
silo SPA pages, plugin retirement, cutover. Each is its own sub-plan
(2–6).
**Placeholder scan:** None. Every step has the actual command, the
actual SQL, or the actual Go to write. Conditional language is confined
to Task 4 (which is explicit about its condition and what to do if it
fires).
**Type consistency:** The fake `SettingsReader` interface in Task 6's
test matches the production interface defined in `service.go`. Method
name `GetString`, signature `(ctx context.Context, key string) (string, error)` — consistent across both files.
**Note on migration numbering:** Tasks 2–5 originally used numbers `139..142`
based on a snapshot showing `138_search_number_word_normalization.up.sql` as
the highest existing migration. The landed implementation was renumbered to
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
`160_audiobooks_feature_flag`.
docs/superpowers/plans/2026-05-24-audiobooks-absorption-2-scanner.md
+1075
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-24-audiobooks-absorption-3-api-and-frontend.md
+326
View File
@@ -0,0 +1,326 @@
# Audiobooks Absorption — Sub-plan 3: Silo-native API + Frontend (MVP)
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement task-by-task. Steps use checkbox (`- [ ]`) syntax.
**Goal:** Land the minimum viable silo-native audiobooks UI: backend REST endpoints + React pages so a user can browse audiobook libraries scanned by sub-plan 2, see book details with chapters, play with progress saved. Author/series index pages, smart collections, share links, and other features from the spec's "nice-to-haves" list are deferred to a future sub-plan if the demand surfaces.
**Architecture:** Three new Go handler files under `internal/api/handlers/audiobooks_*.go` for list/detail/progress. New `web/src/pages/audiobooks/` directory with three pages: Library, Detail, and an inline player. Hooks under `web/src/hooks/audiobooks/`. The streaming path reuses silo's existing `/api/v1/stream/{session_id}` endpoint — no new transcode code.
**Tech stack:** Go + chi router + pgx for backend; React 19 + TanStack Query + radix-ui + tailwind for frontend.
**Source spec:** `docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md`
**Predecessor:** `docs/superpowers/plans/2026-05-24-audiobooks-absorption-2-scanner.md` (scanner produces `media_items.type='audiobook'`)
---
## File Structure
| Path | C/M | Purpose |
|---|---|---|
| `internal/api/handlers/audiobooks_list.go` | C | `HandleListAudiobooks` — GET /api/v1/audiobooks (paginated, filterable by library) |
| `internal/api/handlers/audiobooks_detail.go` | C | `HandleGetAudiobook` — GET /api/v1/audiobooks/{id} (returns item + chapters + author/narrator + listening progress) |
| `internal/api/handlers/audiobooks_progress.go` | C | `HandleReportAudiobookProgress` — POST /api/v1/audiobooks/{id}/progress |
| `internal/api/router.go` | M | Mount three new routes under `/api/v1/audiobooks/*` |
| `web/src/hooks/audiobooks/useAudiobookLibrary.ts` | C | TanStack Query hook for list |
| `web/src/hooks/audiobooks/useAudiobook.ts` | C | Detail + progress hooks |
| `web/src/pages/audiobooks/AudiobookLibrary.tsx` | C | Grid page (poster + title + author) |
| `web/src/pages/audiobooks/AudiobookDetail.tsx` | C | Detail page (cover, metadata, chapter list, play button) |
| `web/src/pages/audiobooks/AudiobookPlayer.tsx` | C | HTML5 `<audio>` player with chapter nav + progress reporting |
| `web/src/App.tsx` or routing entry | M | Add the three new routes |
| `web/src/components/Sidebar.tsx` (or equivalent) | M | Add "Audiobooks" navigation link when at least one audiobook library exists |
---
## Task 1: Backend — list audiobooks
GET /api/v1/audiobooks?library_id=N&limit=...&offset=... → paginated audiobook list scoped to libraries the user has access to.
**Files:** Create `internal/api/handlers/audiobooks_list.go`, modify `internal/api/router.go`.
### 1.1 Test
Add `internal/api/handlers/audiobooks_list_test.go` with a test that:
- Inserts two `media_items` rows (one `type='audiobook'`, one `type='movie'`)
- Calls the handler with a fake request
- Asserts only the audiobook is returned
If no test harness exists for handler-level tests in `internal/api/handlers/`, search neighbors (`grep -lE 'func.*Handler.*Test' internal/api/handlers/*_test.go`) for the established pattern. If integration testing requires a real DB and no harness exists, write the handler logic + a simple compile-time test only; defer the integration test (DONE_WITH_CONCERNS).
### 1.2 Handler
```go
package handlers
import (
"encoding/json"
"net/http"
"strconv"
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
"github.com/Silo-Server/silo-server/internal/catalog"
)
type AudiobookHandler struct {
Items *catalog.ItemRepository
}
func (h *AudiobookHandler) HandleListAudiobooks(w http.ResponseWriter, r *http.Request) {
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
if limit <= 0 || limit > 200 {
limit = 50
}
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
if offset < 0 {
offset = 0
}
filter := apimw.AccessFilterFor(r)
items, total, err := h.Items.Search(r.Context(), "", []string{"audiobook"}, limit, offset, filter)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks failed")
return
}
resp := struct {
Items []map[string]any `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}{
Items: itemSummariesAsMaps(items),
Total: total,
Limit: limit,
Offset: offset,
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(resp)
}
func itemSummariesAsMaps(items []*models.MediaItem) []map[string]any {
out := make([]map[string]any, 0, len(items))
for _, it := range items {
out = append(out, map[string]any{
"content_id": it.ContentID,
"title": it.Title,
"year": it.Year,
"poster_url": it.PosterPath, // silo's poster column; UI prefixes with media-token base URL
})
}
return out
}
```
Verify `catalog.ItemRepository.Search` matches the signature above by reading `internal/catalog/item_repo.go:739`. Adjust if needed.
### 1.3 Route + commit
In `internal/api/router.go`, find where other v1 routes are mounted and add:
```go
r.Get("/audiobooks", audiobookHandler.HandleListAudiobooks)
```
Wire the handler construction near other handler constructors. The handler needs `catalog.ItemRepository` — already available in the router's deps.
Commit: `feat(audiobooks): list endpoint at GET /api/v1/audiobooks`.
---
## Task 2: Backend — audiobook detail
GET /api/v1/audiobooks/{id} → media_item + media_files (with chapters) + author/narrator from item_people + per-profile listening progress.
### 2.1 Handler
Create `internal/api/handlers/audiobooks_detail.go`. The handler queries:
1. `media_items WHERE content_id = $1 AND type = 'audiobook'`
2. `media_files WHERE content_id = $1` (returns paths + chapters JSONB)
3. `item_people JOIN people ON ... WHERE content_id = $1 AND kind IN (7, 8)`
4. `user_watch_progress WHERE content_id = $1 AND profile_id = $current`
Reuse existing repo methods where they exist. If not, add inline SQL.
Response shape:
```json
{
"audiobook": { "content_id": "...", "title": "...", "year": 2024, "overview": "...", "poster_url": "..." },
"author": "Test Author",
"narrator": "Test Narrator",
"files": [
{ "id": 123, "path": "/...", "duration_seconds": 3600, "chapters": [ {"index": 0, "title": "Intro", "start_seconds": 0, "end_seconds": 27.9} ] }
],
"progress": { "position_seconds": 1842.5, "updated_at": "2026-05-24T..." }
}
```
### 2.2 Route + commit
In `router.go`: `r.Get("/audiobooks/{id}", audiobookHandler.HandleGetAudiobook)`.
Commit: `feat(audiobooks): detail endpoint at GET /api/v1/audiobooks/{id}`.
---
## Task 3: Backend — progress endpoint
POST /api/v1/audiobooks/{id}/progress with body `{"position_seconds": 1842.5, "media_file_id": 123}`. UPSERTs `user_watch_progress`.
### 3.1 Handler
Create `internal/api/handlers/audiobooks_progress.go`. Body is JSON; handler parses, validates `position_seconds >= 0`, then upserts into `user_watch_progress` keyed on `(user_id, profile_id, media_item_id)`.
Reuse silo's existing progress-write helper if one exists (grep `INSERT INTO user_watch_progress` or `UpsertWatchProgress`). If not, write inline SQL using the existing DB pool from handler deps.
### 3.2 Route + commit
`r.Post("/audiobooks/{id}/progress", audiobookHandler.HandleReportAudiobookProgress)`.
Commit: `feat(audiobooks): progress endpoint at POST /api/v1/audiobooks/{id}/progress`.
---
## Task 4: Frontend — TanStack Query hooks + types
**Files:**
- Create: `web/src/lib/audiobooks/types.ts`
- Create: `web/src/hooks/audiobooks/useAudiobookLibrary.ts`
- Create: `web/src/hooks/audiobooks/useAudiobook.ts`
- Create: `web/src/hooks/audiobooks/useReportAudiobookProgress.ts`
### 4.1 Types
```ts
// web/src/lib/audiobooks/types.ts
export interface AudiobookSummary {
content_id: string;
title: string;
year: number;
poster_url: string | null;
}
export interface AudiobookListResponse {
items: AudiobookSummary[];
total: number;
limit: number;
offset: number;
}
export interface AudiobookChapter {
index: number;
title: string;
start_seconds: number;
end_seconds: number;
}
export interface AudiobookFile {
id: number;
path: string;
duration_seconds: number;
chapters: AudiobookChapter[];
}
export interface AudiobookProgress {
position_seconds: number;
updated_at: string;
}
export interface AudiobookDetail {
audiobook: { content_id: string; title: string; year: number; overview: string; poster_url: string | null };
author: string;
narrator: string;
files: AudiobookFile[];
progress: AudiobookProgress | null;
}
```
### 4.2 Hooks
Use TanStack Query patterns silo already uses (grep `useQuery` for examples). Cache keys: `["audiobooks", "library", libraryId]` and `["audiobook", id]`. Progress mutation invalidates the detail query on success.
### 4.3 Commit
Commit: `feat(audiobooks): frontend types and TanStack Query hooks`.
---
## Task 5: Frontend — Audiobook Library page
**Files:**
- Create: `web/src/pages/audiobooks/AudiobookLibrary.tsx`
Grid of audiobook cards (poster + title + author). Use silo's existing card / grid components — grep `web/src/components/` for `MediaCard` or similar. Infinite scroll via `useInfiniteQuery` would be ideal but a simple paginated load-more is fine for MVP.
Route URL: `/audiobooks` (or `/audiobooks/library/:id` if there are multiple libraries). For MVP, single library route `/audiobooks` is fine.
Commit: `feat(audiobooks): library grid page`.
---
## Task 6: Frontend — Audiobook Detail page
**Files:**
- Create: `web/src/pages/audiobooks/AudiobookDetail.tsx`
Layout: cover image (poster) + metadata (title, author, year, overview) + chapter list + play button. Chapter list is collapsible; clicking a chapter starts playback at that chapter's start_seconds.
Route URL: `/audiobooks/book/:id`.
Commit: `feat(audiobooks): detail page with chapter list`.
---
## Task 7: Frontend — Audiobook Player
**Files:**
- Create: `web/src/pages/audiobooks/AudiobookPlayer.tsx` (or `web/src/player/AudiobookPlayer.tsx`)
HTML5 `<audio>` element. The stream URL for an audiobook file is silo's existing `/api/v1/stream/{session_id}` — call POST /api/v1/playback/session to start a session (existing silo endpoint), then point the `<audio src>` at the stream URL. Position updates throttled to every 5-10 seconds + on pause/seek.
Controls: play/pause, skip-30s-forward, skip-30s-back, playback rate select (0.5×, 0.75×, 1×, 1.25×, 1.5×, 2×, 3×), chapter list panel.
If silo's existing stream-session-start endpoint shape isn't obvious, grep for `playback/session` and copy the pattern from the video player.
Commit: `feat(audiobooks): HTML5 audio player with chapter navigation`.
---
## Task 8: Navigation + routing integration
**Files:**
- Modify: `web/src/App.tsx` (or wherever routes are registered) — add three new routes
- Modify: `web/src/components/Sidebar.tsx` (or equivalent) — add "Audiobooks" nav link
The sidebar link should only show when at least one audiobook library exists. Use a `useAudiobookLibrary` query with a short staleTime to gate visibility, OR a separate `useLibrariesOfKind("audiobooks")` hook.
Commit: `feat(audiobooks): wire navigation and routes`.
---
## Task 9: Build, lint, smoke
```bash
cd /opt/silo-server
go build ./...
go test ./internal/api/handlers/audiobooks_*_test.go ./internal/scanner/... ./internal/models/... ./internal/audiobooks/...
go vet ./...
cd web && pnpm run lint && pnpm run format:check && cd ..
make build
```
Rebuild silo image + force-recreate container:
```bash
sudo docker build -t silo:latest /opt/silo-server
sudo docker compose -p silo-prod up -d --force-recreate silo
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
curl -sS -o /dev/null -w "GET /api/v1/audiobooks -> %{http_code}\n" http://localhost:8090/api/v1/audiobooks
```
If any sweep changes (gofmt, prettier) were made, commit them with `chore(audiobooks): sweep formatting`.
---
## Risks
- Frontend tasks depend on silo's existing component shapes. Subagents will need to grep `web/src/components/` for the right patterns. If they invent ad-hoc components, ask them to use existing ones.
- The progress endpoint UPSERT shape depends on `user_watch_progress` PK shape — which discovery D4 confirmed is `(user_id, profile_id, content_id)`. Confirm and use exactly that ON CONFLICT target.
- Audiobook libraries may not yet exist on the test stack. To smoke-test, an operator must manually update a `media_folders.type` to `'audiobooks'` and trigger a rescan after the sub-plan 2 scanner branch lands.
docs/superpowers/plans/2026-05-24-audiobooks-absorption-4-abs.md
+148
View File
@@ -0,0 +1,148 @@
# Audiobooks Absorption — Sub-plan 4: ABS-compat REST + Socket.io
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. Steps use checkbox (`- [ ]`) syntax.
**Goal:** Port the audiobookshelf-compatible REST + Socket.io surface from `silo-plugin-audiobooks/internal/abs/` and `silo-plugin-audiobooks/internal/abssocket/` into `internal/audiobooks/abs/` and `internal/audiobooks/abssocket/`. **Preserve the plugin's file layout** but **rewrite every SQL query** against silo's existing tables (`media_items`, `media_files`, `episodes`, `user_watch_progress`, `user_playback_sessions`, `abs_sessions`, `people`, `item_people`). No new tables.
**Architecture:** Stage-based port. Support files first, then auth, then file serving, then progress, then author/series, then Socket.io, then router wiring. Out-of-scope features (bookmarks, smart collections, share links, playlists, podcasts/RSS in this sub-plan) are dropped — files containing them are not copied, or are copied with the unsupported handlers stubbed to 501 Not Implemented.
**Source plugin:** `/opt/silo_plugins/silo-plugin-audiobooks/internal/{abs,abssocket}/`
---
## Stage 1: Support files
**Files to copy and adapt:**
- `mount.go` — package mount + route registration shape
- `handler.go` — top-level handler type + dependency wiring
- `filter.go` — generic helpers
- `access_log.go` — request-log helper
- `minified.go` — generic helpers (likely small)
**Adapt steps:**
1. Replace import paths: `github.com/RXWatcher/silo-plugin-audiobooks/...` → `github.com/Silo-Server/silo-server/internal/audiobooks/...`
2. Replace plugin SDK references with direct silo equivalents:
- Plugin's `*pgx.Pool` field on Handler → silo's `*pgxpool.Pool` (same type via different import alias)
- Plugin's `Logger` → silo's `log/slog` package-level functions
- Plugin's config types → struct fields on `audiobooks.Service` (extended)
3. Remove any code that talks to the plugin SDK runtime (plugin proxy callbacks etc.)
4. Get `go build ./internal/audiobooks/abs/...` to compile cleanly even if no routes are wired yet.
Commit: `feat(audiobooks): port ABS handler scaffolding (mount/handler/filter)`.
---
## Stage 2: Auth + JWT
**Files:**
- `jwt.go` — token generation + validation
- `login_internal_test.go` (keep as test) + `login_ratelimit.go`
- Whatever the actual `login.go` file is called (look for the login handler — likely inside `handler.go` or a dedicated file)
**Adapt:**
- The plugin's login handler posts credentials to silo's `/api/v1/auth/login` over HTTP. **In silo's in-process port, replace this HTTP round-trip with a direct call to silo's auth service** (find via `grep -nE 'auth.*Authenticate|auth.*Login' /opt/silo-server/internal/auth/*.go`). The plugin's network call is no longer needed because the code is now in-process.
- ABS sessions are stored in `abs_sessions` (migration 139 from sub-plan 1).
- JWT signing secret: store in `server_settings` under key `audiobooks.abs.jwt_secret` (auto-generated on first request if missing).
Commit: `feat(audiobooks): port ABS auth (JWT + login bridge)`.
---
## Stage 3: File serving + play session
**Files:**
- `file_handler.go` — handles ABS GET requests for audio file streams
- `play_response.go` — issues a play session to the client (URL + token)
**Adapt:**
- Plugin's `bookref` lookup table (linking ABS-style IDs to backend file IDs) → use silo's `media_files.id` directly. ABS-side item ID = silo's `content_id`; ABS-side file ID = silo's `media_files.id`.
- The stream URL ABS clients expect must point back at silo. Issue them a `/api/v1/direct-download?file_id={id}` URL signed with silo's existing media-token signer (grep `mediatoken` in `internal/`).
Commit: `feat(audiobooks): port ABS file handler + play session`.
---
## Stage 4: Progress tracking
**Files:**
- `progress_internal_test.go` (test, port)
- Whatever file owns the `/api/me/progress` ABS endpoint family — look in `handler.go` or grep ABS progress route paths
- `continue_listening.go`
**Adapt:**
- All progress reads/writes hit silo's `user_watch_progress` keyed on `(user_id, profile_id, content_id)`. Sub-plan 3's progress handler did this already — copy the patterns.
Commit: `feat(audiobooks): port ABS progress + continue listening`.
---
## Stage 5: Author / Series / Browse
**Files:**
- `author_series_handler.go`
- `collapse.go`
**Adapt:**
- Author rows come from silo's `people` + `item_people` (kind=7).
- Series grouping: if silo doesn't have an explicit "audiobook series" concept, use `media_items.title` prefix matching (e.g., books named "Foundation #1", "Foundation #2") for MVP. Document the heuristic.
Commit: `feat(audiobooks): port ABS author/series browse`.
---
## Stage 6: Socket.io
**Files:**
- `abssocket/server.go` + `abssocket/server_test.go`
**Adapt:**
- Reuse silo's `gorilla/websocket` upgrader. The plugin's standalone-listener escape hatch is no longer needed — mount under `/abs/socket.io/` on silo's main listener.
- Redis adapter behavior preserved if `REDIS_URL` is set (silo always has one in the docker setup).
Commit: `feat(audiobooks): port ABS Socket.io endpoint`.
---
## Stage 7: Router wiring
**Files:**
- Modify `internal/api/router.go` — mount `audiobookABSHandler` under `/abs/*` and the small set of `/api/*` legacy paths ABS clients hardcode
- Modify `internal/audiobooks/service.go` — add `ABSHandler` field, wire it from `audiobooks.New(...)`
- Modify `cmd/silo/main.go` — construct the ABS handler and pass it into the audiobooks service
**Routes to expose** (the minimum ABS-required set):
- `POST /abs/login` (login flow)
- `GET /abs/api/me` (profile)
- `GET /abs/api/libraries` (library list)
- `GET /abs/api/libraries/{id}/items` (browse)
- `GET /abs/api/items/{id}` (item detail)
- `GET /abs/api/items/{id}/file/{ino}` (stream)
- `POST /abs/api/me/progress/{libraryItemId}` (progress)
- `GET /abs/socket.io/` (Socket.io)
For ABS-legacy hard-coded paths that don't fit under `/abs/`: register them explicitly with the ABS auth middleware so they don't collide with silo's existing `/api/v1/*`.
Build + smoke:
```bash
sudo docker build -t silo:latest /opt/silo-server
sudo docker compose -p silo-prod up -d --force-recreate silo
curl -sS -o /dev/null -w "POST /abs/login -> %{http_code}\n" -X POST -H 'Content-Type: application/json' -d '{}' http://localhost:8090/abs/login
```
Expected: 400 (bad request body) or 401 (validation failed), NOT 404.
Commit: `feat(audiobooks): wire ABS routes into main router`.
---
## Stage 8: Build + smoke + lint sweep
Single final task. Same shape as sub-plan 2 Task 10.
---
## Risks
- The plugin's code expects a specific database schema. Some plugin queries may have no clean silo-table equivalent (e.g., the plugin's `bookref` mapping table). In that case, stub the handler to return 501 + log + flag in PR notes — don't add new tables.
- ABS clients have undocumented expectations (CORS headers, specific error shapes, etc.). Smoke testing against a real ABS client is the only way to find these. Defer to sub-plan 6's manual verification.
- Each stage commits independently and produces partial functionality (e.g., after stage 2 you can log in but can't list libraries). That's intentional — clients won't connect end-to-end until stage 7.
docs/superpowers/plans/2026-05-24-audiobooks-absorption-5-podcasts.md
+35
View File
@@ -0,0 +1,35 @@
# Audiobooks Absorption — Sub-plan 5: Podcasts (RSS feed refresher)
**Goal:** Port the plugin's `podcastfeed` package as a first-party scheduled task in silo, so RSS-subscribed podcasts get their episode lists kept up to date.
**Source plugin:** `../silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
## Tasks
### Task 1: Port the refresher package
- Copy `refresher.go` + `refresher_test.go` from plugin to `internal/audiobooks/podcastfeed/`
- Update imports (drop plugin SDK; use silo's `pgxpool` + `log/slog`)
- Rewrite SQL queries against silo's `podcast_feeds` table (created in sub-plan 1 migration 140), `media_items` (type='podcast'), and `episodes`
- Drop the plugin's `presentation_libraries` reference; subscriptions are tracked by `podcast_feeds.media_item_id`
### Task 2: Wire as scheduled task
- Discovery D8 documented silo's task registration pattern: `taskMgr.Register(tasks.NewXxxTask(...))` in `cmd/silo/main.go`
- Add `tasks.NewSyncPodcastFeedsTask(refresher)` and register
- Default interval: 10 minutes (matches plugin)
### Task 3: ABS podcast endpoints (optional)
The plugin's `abs/podcast.go` + `abs/rss_feed_handler.go` + `abs/podcast_handler.go` expose:
- `GET /abs/api/libraries/{id}/podcasts/{podcastId}/episodes/{episodeId}`
- `POST /abs/api/libraries/{id}/podcasts` (add subscription)
- `GET /abs/api/podcast/feeds` (admin RSS feed browse)
If reach is required for ABS clients, port these. Otherwise stub 501 and ship. For initial release, stub.
### Task 4: Build + smoke
- Verify scheduled task registers on startup (`docker logs` shows the task)
- Verify nothing else breaks
## Risks
- Plugin's RSS parser may depend on third-party packages not in silo's go.mod. Check go.mod after porting and add if needed.
- `episodes` table in silo has a different shape than the plugin assumed. Re-read the plugin's INSERT statements and adapt.
docs/superpowers/plans/2026-05-26-abs-bookmarks-implementation.md
+1596
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-26-abs-collections-playlists-implementation.md
+4087
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-26-abs-phase-0-login-and-critical-fixes.md
+1778
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-26-abs-phase1-closeout-implementation.md
+2068
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-26-abs-smart-collections-implementation.md
+2531
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-05-27-audiobook-catalog-filter-fields.md
+187
View File
@@ -0,0 +1,187 @@
# Audiobook-native catalog filter fields
**Status:** Draft — not yet approved or executed.
**Context.** The audiobooks library page currently inherits the catalog's
movie/TV filter set. We've already (a) wired `libraryType="audiobook[s]"`
into the sort-relevance scope so video-only sorts (resolution, IMDb/RT
ratings, content rating) drop out, and (b) gated the obviously-irrelevant
video-only filter sections (Director, Writer, Producer, Studio, Network,
Video Quality) in `CollectionGuidedRulesEditor`. What remains is
**affirmative** support for audiobook-native filter dimensions: **author**,
**narrator**, **series**.
Commands assume the repository root is the cwd.
## Goal
Surface three new filter dimensions on the audiobook library page:
1. **Author** — `item_people.kind = PersonKindAuthor` (7).
2. **Narrator** — `item_people.kind = PersonKindNarrator` (8).
3. **Series** — `audiobook_series.series_name` (free-text or distinct list).
The user should be able to:
- See per-dimension counts/options in the filter sheet (populated from the
current scope, like Genres/Studios already do).
- Add a rule via the guided editor that filters items by exact author /
narrator / series.
## Non-goals
- Series **ordering** (we already have `audiobook_series.series_index`).
Sort by series is a separate piece of work; this plan focuses on
filtering only.
- Author/narrator typeahead UI improvements beyond the existing
`PersonSearchSelect` (reuse it with kind-scoped lookups).
## Backend
### `internal/catalog/catalog_resolver.go`
`CatalogFiltersResult` currently has `Genres`, `Studios`, `Networks`,
`Countries`, `OriginalLanguages`, `ContentRatings`, `Resolutions`,
`AudioLanguages`, `SubtitleLanguages`. Add:
```go
Authors []string `json:"authors,omitempty"`
Narrators []string `json:"narrators,omitempty"`
Series []string `json:"series,omitempty"`
```
In `listFiltersForSource`, add three parallel facet queries:
- `listDistinctPeopleByKind(ctx, scope, models.PersonKindAuthor)` — joins
`item_people` + `people` on the result set defined by the current
request scope (library_ids, media_scope=audiobook, etc.) and returns
distinct `people.name`.
- Same for `PersonKindNarrator`.
- `listDistinctAudiobookSeriesNames(ctx, scope)` — distinct
`audiobook_series.series_name` for the scope, joined via
`audiobook_series.content_id = media_items.content_id`.
All three need to respect the access filter the caller passes in (mirror
how Genres/Studios already gate by access).
### `internal/api/handlers/catalog.go`
Extend `catalogFiltersResponse` and `HandleGetCatalogFilters` to emit the
new fields. Mirror the pattern used for the existing ones — no flag
gating, audiobook filters should always be included when present.
### `internal/catalog/query_builder.go`
Three new field names: `author`, `narrator`, `series`. The first two
reuse `buildPersonClause` with the appropriate `PersonKind`. `series`
needs a small new clause:
```go
case "series":
// EXISTS (SELECT 1 FROM audiobook_series s
// WHERE s.content_id = media_items.content_id
// AND lower(s.series_name) = lower($N))
```
Register the three field names in `catalogQueryRuleFields` so the parser
accepts them. None of them are personalized — they don't go into
`catalogPersonalRuleFields`.
### Tests
- `internal/catalog/catalog_resolver_test.go` — extend with audiobook
scope: assert authors/narrators/series come back, scoped to libraries
the test user has access to.
- `internal/catalog/query_builder_test.go` — add cases for `author`,
`narrator`, `series` rules; verify the emitted SQL clauses match the
expected shape.
## Frontend
### `web/src/api/types.ts`
Extend `CatalogFiltersResponse` (and any `ItemFiltersResponse` base, if
shared) with optional `authors?: string[]`, `narrators?: string[]`,
`series?: string[]`.
Extend `QueryRule['field']` to include `'author' | 'narrator' | 'series'`
if it's a literal union (or no-op if it's `string`).
### `web/src/components/collections/CollectionGuidedRulesEditor.tsx`
Add to `GuidedFormState`:
- `author: string`
- `narrator: string`
- `series: string`
Update `queryDefinitionToGuidedState` / `guidedStateToQueryDefinition` to
round-trip the three rules (mirror how `actor` / `director` are wired).
In the audiobook-only render path (already gated via `isAudiobookLibrary`),
add a new section *above* the country row:
```tsx
<div className="grid gap-4 md:grid-cols-2">
<Author picker /> {/* PersonSearchSelect with kind="author" */}
<Narrator picker /> {/* PersonSearchSelect with kind="narrator" */}
</div>
<div className="grid gap-4 md:grid-cols-2">
<Series picker /> {/* SearchableSelect over filters.series */}
</div>
```
`PersonSearchSelect` likely needs a new `kind` prop so it can scope its
backend lookup to authors or narrators. If today it queries all kinds,
we'll add a query param + handler-side filter.
### `ActiveFilterBadges` + `catalogFilterBadges`
Add badge rendering for the three new fields so they appear in the
selected-filters chip row above the editor.
### Tests
- `web/src/components/collections/CollectionGuidedRulesEditor.test.tsx`
— render with `libraryType="audiobooks"` and a filters payload
containing authors/narrators/series; assert all three sections appear
and update the query definition correctly.
## Migration / data
No schema changes required. `PersonKindAuthor=7` and `PersonKindNarrator=8`
already exist in `internal/models/media.go`. `audiobook_series` already
holds series names (migration 145). The scanner is already writing both.
## Risk
- **Filter facet performance.** Three new distinct-value queries per
`/catalog/filters` request. The existing facet queries use the same
`facetFetcher` infrastructure, so they'll inherit the same query-time
budget. For an audiobook library on the order of a few thousand items
this should be fine; if it's not, we can cache author/narrator lookups
more aggressively (they change rarely).
- **PersonSearchSelect kind scoping.** If the component today emits a
single combined kind=any search, we need to add kind-filtered variants
without breaking the existing actor/director/writer/producer pickers.
- **Series name normalization.** Backfill at migration 145 used regex
parsing; some items may have inconsistent casing or whitespace.
Distinct facet query should `TRIM(series_name)` and present `LOWER`
for matching; otherwise duplicates show up in the picker.
## Rollout
This is gated entirely behind `libraryType === "audiobook[s]"` on the
frontend, and the new filter fields are additive on the backend. No
feature flag needed. Ship in two PRs if reviewer prefers:
1. **Backend** — filter response fields + query-builder clauses + tests.
2. **Frontend** — types + GuidedFormState + editor sections + tests.
Or a single PR if the diff stays reviewable (~600 lines including tests).
## Verification
- Add new tests as above; `make lint` and `cd web && pnpm vitest run`
should pass.
- Manually: open the audiobook library filter sheet, pick an author and
a narrator, confirm the result set narrows. Pick a series, confirm
same.
docs/superpowers/plans/2026-05-27-audiobook-series-data-cleanup.md
+123
View File
@@ -0,0 +1,123 @@
# `audiobook_series` data cleanup
**Status:** Draft. Findings + options; no implementation yet.
## Problem
Migration `145_audiobook_series.up.sql` (which created the
`audiobook_series` table) shipped with a regex backfill that
over-matched. On this server's 220k-book audiobook library it produced
**162,322 distinct series names**, of which **141,591 (87%) are
singletons** — books that are the only "member" of their named series.
Spot checks make the failure mode obvious: rows like
- `series_name = "Computer Programming: This Book Includes: SQL, …"` for a standalone book whose subtitle happened to contain `"#2022 Version"`
- `series_name = "Reparación de crédito [Credit Repair]"` (the title verbatim)
- `series_name = "Cómo Atraer a las Mujeres … [How to Attract Women …]"` (the title verbatim)
The regex (`'^.+[^\s-]\s+\d+(?:\.\d+)?\s*-\s*.+$'`) treats any numeric
chapter / edition / part token followed by ` - ` as series-N-of-book
syntax. The real series (Star Wars, Warhammer 40k, Animorphs, Redwall,
*In Death*, etc.) live in the 733-row "large series" bucket; the
9,075 pairs and 10,923 small clusters mix real and false-positive.
The cap added in `b95494c` keeps the user-facing dropdown manageable
(top 1000 alphabetically), but the underlying data is still polluted —
sorting by series, the Series filter dropdown, and any future
"recommended" surfacing of series will all surface noise until the
backfill is repaired.
Commands assume the repository root is the cwd.
## Options
### A — delete singletons (destructive)
```sql
DELETE FROM audiobook_series
WHERE series_name IN (
SELECT series_name FROM audiobook_series GROUP BY series_name HAVING COUNT(*) = 1
);
```
- **Pros:** simple, removes the bulk of the noise in one shot, no
schema change.
- **Cons:** also deletes legitimate "first book of a not-yet-complete
series" rows. Reversal requires a re-scan.
### B — filter singletons at query time, leave data alone
In `listDistinctAudiobookSeriesWithSource`, group by series name and
`HAVING COUNT(*) >= 2`.
- **Pros:** non-destructive; the table keeps full information for
detail-page lookups and series-detail surfaces.
- **Cons:** hides legitimate one-book-only series from the filter
dropdown. Series-detail and sort behaviour unchanged (still surface
singletons). The 11.7 MB payload was already capped to 1000 in
`b95494c`, so the size win from this option is small.
### C — mark backfill provenance, delete only backfilled singletons
Add a `source` column to `audiobook_series` (`'scanner' | 'backfill'`)
in a new migration. Migration 145's backfill rows get `source =
'backfill'`; scanner-written rows get `'scanner'`. Then delete only the
backfilled singletons.
- **Pros:** distinguishes legitimate from spurious without guessing.
- **Cons:** retroactive — we'd have to assume all current rows are
backfill (which they mostly aren't anymore — the scanner has been
writing too). Without a marker added at backfill time, can't tell
what's what.
### D — wipe + re-scan (clean slate)
```sql
TRUNCATE audiobook_series;
```
Then trigger a full library re-scan so the scanner writes only the
real series_name values it extracts from tags.
- **Pros:** the cleanest end state; the scanner is authoritative.
- **Cons:** long re-scan window (the user already noticed 219k books
is slow to scan); any books not currently scannable lose their
series info entirely. Destructive.
## Recommendation
**Option B as a quick win** — non-destructive, ships as a one-line SQL
change in the facet helper, makes the Series filter dropdown useful
immediately. The dropdown stops showing 87% noise.
**Option D as the eventual fix** — once the scanner has been verified
to write series_name authoritatively (including the series_index from
real tags), `TRUNCATE audiobook_series` + re-scan is the right end
state. Need to confirm scanner behaviour before pulling the trigger.
Option A is too blunt (drops real first-books-of-series). Option C is
over-engineered without a marker added at backfill time.
## What this plan does not cover
- Whether the scanner currently writes `audiobook_series` rows
authoritatively for every audiobook it touches, or only when tags
contain explicit series metadata. Need to check
`internal/scanner/audiobook.go` (or wherever the audiobook_series
upsert lives) before proposing option D as the long-term path.
- Whether *Search Series* (the user-facing series-detail page, if
any) would also benefit from the same cleanup.
- Sort-by-series on the library page — even with option B in place,
the sort still groups by every series_name including singletons,
so books in fake "series" appear under their fake series name.
Acceptable for now; revisit alongside option D.
## Verification (once an option is picked)
- Re-run the data sanity query in this doc's intro and confirm the
singleton count dropped where expected.
- Open the Series filter dropdown on the audiobook library page and
confirm only multi-book series show up.
- `go test ./internal/catalog/...` + `cd web && pnpm vitest run` still
pass.
docs/superpowers/plans/2026-05-27-catalog-facet-typeahead.md
+143
View File
@@ -0,0 +1,143 @@
# Server-side typeahead for catalog facets
**Status:** Draft. Architecture sketch; no implementation yet.
## Why
`b95494c` capped `/api/v1/catalog/filters` facet responses to 1000
distinct values each, which fixed an 11.7 MB payload on the audiobook
library (88k authors, 92k narrators, 161k series). The cap is a
stopgap — a user looking for an author past the first 1000
alphabetically can't find them through the dropdown.
Parallel context: a separate investigation (transcript in the
2026-05-27 sessions) compared how `librarymanagerre`, `booklore-ng`,
and `audiobookshelf` paginate large libraries. The cursor +
virtual-scroll camp (librarymanagerre, booklore-ng's
`@tanstack/react-virtual`) scales to 250k+ items by never sending the
client a full list; the eager-shelves camp (audiobookshelf web +
mobile) renders one DOM div per item and falls over above ~30–50k.
The 11.7 MB facet issue here is the same shape on a different surface
— "too much data shipped to the client at once."
The cleanest fix is **server-side typeahead**: the client sends a
search prefix as the user types, the server returns the top N matches
for that prefix.
Commands assume the repository root is the cwd.
## Surface
### `/api/v1/catalog/filters/search`
New endpoint, scoped per facet, query parameters mirror
`/api/v1/catalog/filters` (`source`, `library_id`, etc.) plus:
- `facet=author|narrator|series|studio|network|country|genre|original_language|content_rating`
- `q=<prefix>` — the user's typed prefix (1-64 chars, trimmed)
- `limit=<N>` — capped at e.g. 50
Response:
```json
{ "matches": ["Brandon Sanderson", "Brandon Sanderson & Steven Erikson", ...], "has_more": false }
```
Returns up to `limit` matches sorted by:
1. exact-prefix match first (`q` matches the start of the value)
2. then case-insensitive substring match
3. then alphabetical
`has_more` is true when the underlying result set was truncated.
### `/api/v1/catalog/filters` keeps the current shape
No change to the existing endpoint — it still returns the capped top
1000 per facet for the initial dropdown render. The typeahead surface
takes over once the user starts typing.
## Backend changes
### `internal/catalog/catalog_resolver.go`
New method `SearchFacetWithOptions(ctx, req, access, facet, prefix,
limit)` that:
1. Resolves the same access/scope as `ListFiltersWithOptions`.
2. Dispatches to a facet-specific helper based on `facet`.
3. Returns the matches + has_more flag.
Add to `facetFetcher` interface:
```go
SearchPeopleByKind(ctx, kind models.PersonKind, filters BrowseFilters, baseRelation, mediaScope, prefix string, limit int) ([]string, bool, error)
SearchAudiobookSeries(ctx, filters BrowseFilters, baseRelation, mediaScope, prefix string, limit int) ([]string, bool, error)
SearchDistinctArrayColumn(ctx, column string, filters BrowseFilters, baseRelation, mediaScope, prefix string, limit int) ([]string, bool, error)
SearchDistinctScalarColumn(ctx, column string, filters BrowseFilters, baseRelation, mediaScope, prefix string, limit int) ([]string, bool, error)
```
Each helper runs the existing facet SQL with an added
`WHERE LOWER(<value>) LIKE LOWER($N || '%')` (prefix match) and
`LIMIT N+1` (so we can detect has_more by checking if the result set
exceeds N).
### `internal/api/handlers/catalog.go`
New handler `HandleCatalogFacetSearch` mounted at
`/api/v1/catalog/filters/search`.
## Frontend changes
### `web/src/components/ui/searchable-select.tsx` (or a new variant)
`SearchableSelect` today does client-side filtering over the full
`options` array. Replace it for the high-cardinality facets with a
debounced server search:
- Type `<= 100ms` of inactivity → fire `/catalog/filters/search?facet=author&q=...`
- Reset focus + cancel in-flight when the user keeps typing
- Show the first 50 matches; "show more" disabled (the user types more
to narrow further)
The existing `SearchableSelect` stays for low-cardinality facets
(genres, content_ratings, etc.) where the initial 1000 is plenty.
### `web/src/components/collections/CollectionGuidedRulesEditor.tsx`
Author / Narrator / Series sections switch to the new typeahead-backed
select. Studio / Network / Country can stay on `SearchableSelect`
(low-cardinality) or migrate later for consistency.
## Tests
- Backend: search SQL emits the LIKE prefix + LIMIT N+1, has_more
semantics, access filter still gates the result set.
- Frontend: typeahead component debounces, cancels stale requests,
renders empty / loading / no-results states.
## Migration
None — additive endpoint. Frontend can be rolled out incrementally:
audiobook-only sections first (since they're the worst offenders),
others later.
## What this plan does not cover
- The audiobookshelf-app pagination question from the separate
transcript. That's a different surface (the ABS-compat
`/abs/api/libraries/{id}/items` endpoint on `:13378`), not the silo
native catalog. Same architectural family but the patch lives in
`internal/audiobooks/abs/libraries_handler.go`, not catalog.
- A general client-side virtual scroll for the library grid itself.
That's the cousin problem ("250k books rendered as 125k shelf
divs"); cross-applies but is its own work.
## Risk / rollout
- New endpoint is opt-in for the frontend — existing callers keep
using the bulk `/filters` response.
- Typeahead latency budget: each keystroke fires one DB query against
the same indexes the bulk facets already use. Should be fast
(`item_people(content_id, kind)` covers people lookups,
`audiobook_series(content_id)` covers series, and a `LOWER()`
functional index already exists on
`audiobook_series_name_lower`). Verify with EXPLAIN before shipping.
docs/superpowers/plans/2026-05-27-collections-unify-1-schema.md
+314
View File
@@ -0,0 +1,314 @@
# Collections Unification — Sub-project 1: Schema Migration
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Land migration 156 — extend `user_personal_collections` to admit `'playlist'` and `'smart'` types, add nullable `sub_item_id` to its items table, add `media_types` filter to `page_sections`, move the single existing `abs_playlists` row into the canonical store, and drop the five `abs_*` collection tables.
**Architecture:** Pure schema migration + one-row data move. No application code changes. The follow-up sub-projects 2–4 wire silo to the new schema; this one is just the storage foundation. Hard cutover — the `abs_*` tables are near-empty (1 playlist row, 0 of everything else) so rollback risk is minimal.
**Tech Stack:** PostgreSQL 18, raw SQL migrations under `migrations/`, no Go code in this sub-project.
**Commands assume the repository root is the cwd.**
**Source spec:** `docs/superpowers/specs/2026-05-27-unified-audiobook-collections-design.md` §4.1, §4.4, §5.
---
## File map
**Create:**
- `migrations/156_unify_user_collections.up.sql`
- `migrations/156_unify_user_collections.down.sql`
**No Go code modified in this sub-project.** Application code keeps querying the (now-dropped) `abs_*` tables until sub-projects 3 and 4 land, which means **silo will break in dev/staging until sub-project 3 ships.** Land this sub-project and sub-project 3 together (single MR) if a production deploy is imminent. Otherwise it's safe to ship in isolation on a feature branch that hasn't been merged yet.
---
## Task 1: Up-migration body
**Files:**
- Create: `migrations/156_unify_user_collections.up.sql`
- [ ] **Step 1: Write the migration file**
```sql
-- Unify user-owned lists into user_personal_collections.
--
-- See docs/superpowers/specs/2026-05-27-unified-audiobook-collections-design.md
-- for design rationale.
-- 1. Sub-item granularity column on the canonical items table.
-- Empty string for whole-item entries; populated for podcast-episode
-- playlist entries (sub_item_id == abs_playlist_items.episode_id).
ALTER TABLE user_personal_collection_items
ADD COLUMN sub_item_id text NOT NULL DEFAULT '';
-- 2. Widen the collection_type domain. The pre-existing CHECK (if any)
-- only admits 'manual' and 'synced'.
ALTER TABLE user_personal_collections
DROP CONSTRAINT IF EXISTS user_personal_collections_type_check;
ALTER TABLE user_personal_collections
ADD CONSTRAINT user_personal_collections_type_check
CHECK (collection_type IN ('manual', 'synced', 'playlist', 'smart'));
-- 3. Move the existing abs_playlists row(s) into the canonical store.
-- is_public maps to is_shared; profile_id (uuid) is stringified.
INSERT INTO user_personal_collections
(id, user_id, profile_id, name, description, collection_type,
is_shared, created_at, updated_at, creator_profile_id)
SELECT
id,
user_id,
COALESCE(profile_id::text, ''),
name,
description,
'playlist',
is_public,
created_at,
updated_at,
COALESCE(profile_id::text, '')
FROM abs_playlists;
INSERT INTO user_personal_collection_items
(user_id, collection_id, media_item_id, sub_item_id, position, added_at)
SELECT
p.user_id,
i.playlist_id,
i.library_item_id,
i.episode_id,
i.position,
i.added_at
FROM abs_playlist_items i
JOIN abs_playlists p ON p.id = i.playlist_id;
-- 4. Drop the abs_* collection tables. abs_playlist_items has a FK to
-- abs_playlists, so the order matters.
DROP TABLE abs_playlist_items;
DROP TABLE abs_playlists;
DROP TABLE abs_collection_items;
DROP TABLE abs_user_collections;
DROP TABLE abs_smart_collections;
-- 5. Media-type filter on page_sections. Default preserves current
-- behavior (existing rails surface movies+series only).
ALTER TABLE page_sections
ADD COLUMN media_types text[] NOT NULL DEFAULT ARRAY['movie','series'];
```
- [ ] **Step 2: Verify the file compiles as valid SQL**
The repo embeds migrations via `migrations/embed.go`. The migration runner will syntax-check on load.
Run: `go build ./...`
Expected: clean (the embed package compiles fine even if a new file is added).
- [ ] **Step 3: Commit (up only — down comes next task)**
Don't commit yet. Commit happens after Task 2's down-migration so the up/down pair lands atomically.
---
## Task 2: Down-migration body
**Files:**
- Create: `migrations/156_unify_user_collections.down.sql`
The down migration is intentionally lossy in reverse: rolling back loses any `'playlist'`/`'smart'` collections that were created after the up. With the up bringing in 1 row, lossy reverse is acceptable.
- [ ] **Step 1: Find the abs_* CREATE TABLE statements to inline**
Read these files and copy their `CREATE TABLE` bodies (and `CREATE INDEX` statements; **omit** any `DROP TABLE IF EXISTS` boilerplate at the top — the down migration creates from clean state):
```bash
cat migrations/149_abs_user_collections.up.sql
cat migrations/150_abs_collection_items.up.sql
cat migrations/151_abs_playlists.up.sql
cat migrations/152_abs_playlist_items.up.sql
cat migrations/153_abs_smart_collections.up.sql
```
- [ ] **Step 2: Write the down migration**
```sql
-- Reverse migration 156. Lossy in reverse — playlist/smart rows
-- created after the up migration are deleted, not migrated back.
-- 1. Remove the page_sections column.
ALTER TABLE page_sections DROP COLUMN media_types;
-- 2. Recreate the abs_* tables empty. Schemas are inlined from the
-- original up migrations 149–153 — keep identical (column types,
-- constraint names, index names) so any tool keyed off those names
-- sees the same shape.
--
-- BEGIN inlined from migrations/149_abs_user_collections.up.sql
-- <PASTE EXACT CREATE TABLE + CREATE INDEX statements>
-- END inlined
-- BEGIN inlined from migrations/150_abs_collection_items.up.sql
-- <PASTE EXACT CREATE TABLE + CREATE INDEX statements>
-- END inlined
-- BEGIN inlined from migrations/151_abs_playlists.up.sql
-- <PASTE EXACT CREATE TABLE + CREATE INDEX + FK statements>
-- END inlined
-- BEGIN inlined from migrations/152_abs_playlist_items.up.sql
-- <PASTE EXACT CREATE TABLE + CREATE INDEX + FK statements>
-- END inlined
-- BEGIN inlined from migrations/153_abs_smart_collections.up.sql
-- <PASTE EXACT CREATE TABLE + CREATE INDEX statements>
-- END inlined
-- 3. Remove rows we promoted from abs_playlists during the up.
DELETE FROM user_personal_collection_items
WHERE collection_id IN (
SELECT id FROM user_personal_collections
WHERE collection_type IN ('playlist', 'smart')
);
DELETE FROM user_personal_collections
WHERE collection_type IN ('playlist', 'smart');
-- 4. Restore the narrow CHECK constraint.
ALTER TABLE user_personal_collections
DROP CONSTRAINT IF EXISTS user_personal_collections_type_check;
ALTER TABLE user_personal_collections
ADD CONSTRAINT user_personal_collections_type_check
CHECK (collection_type IN ('manual', 'synced'));
-- 5. Drop the sub_item_id column.
ALTER TABLE user_personal_collection_items DROP COLUMN sub_item_id;
```
Replace each `<PASTE EXACT ... statements>` block with the actual SQL from the corresponding `migrations/14X_abs_*.up.sql` file. Do NOT skip any constraint or index — the round-trip test below verifies parity.
- [ ] **Step 3: Verify SQL syntactic validity**
Run: `go build ./...`
Expected: clean.
- [ ] **Step 4: Commit the up+down pair**
```bash
git add migrations/156_unify_user_collections.up.sql migrations/156_unify_user_collections.down.sql
git commit -m "feat(migrations): 156 unify user collections
Adds sub_item_id to user_personal_collection_items, widens
collection_type to include 'playlist' and 'smart', adds media_types
filter to page_sections, moves the existing abs_playlists row into
the canonical store, and drops the abs_* collection tables.
Lays the storage foundation for sub-projects 2-4 to wire the
application against."
```
---
## Task 3: Migration round-trip test
**Files:**
- Create: `migrations/156_unify_user_collections_test.go` (only if a similar pattern exists in the repo)
The repo doesn't have a standard "migration round-trip test" harness today (verify with `find . -name "migration*_test.go" -o -name "*round_trip*_test.go" | head`). If no harness exists, this task is **best-effort manual verification**:
- [ ] **Step 1: Verify a harness pattern exists, or skip to manual verification**
Run:
```bash
find . -name "migration*_test.go" -not -path "*/node_modules/*" 2>/dev/null | head
grep -rln "migrate.Up\|migrate.Down\|migrate.Steps" --include="*_test.go" 2>/dev/null | head
```
If no harness emerges, fall through to Step 2 (manual). Otherwise add a Go test following the existing harness pattern that:
1. Migrates to 155.
2. Inserts a fake `abs_playlists` row (id `test-pl-1`, user 1, name `test`).
3. Inserts a fake `abs_playlist_items` row referencing it.
4. Migrates up to 156.
5. Asserts `user_personal_collections WHERE collection_type='playlist' AND id='test-pl-1'` exists and `user_personal_collection_items WHERE collection_id='test-pl-1'` exists.
6. Migrates down to 155.
7. Asserts the `abs_playlists` table exists and is empty (the row is GONE — down is lossy by design).
- [ ] **Step 2: Manual verification against a throwaway DB**
If no harness, do the following manually once and capture the output in the MR description:
```bash
# 1. Spin up an empty postgres
docker run --rm -d --name pgcheck -e POSTGRES_PASSWORD=x -p 5443:5432 pgvector/pgvector:pg18
sleep 5
# 2. Apply migrations up to 155 (use the existing migrate binary or schema dump)
# Adapt this to whatever the repo uses to run migrations against an arbitrary DB.
# 3. Insert a fake abs_playlists + items row.
PGPASSWORD=x psql -h localhost -p 5443 -U postgres -d postgres -c "
INSERT INTO abs_playlists(id, user_id, name) VALUES ('test-pl-1', 1, 'test');
INSERT INTO abs_playlist_items(playlist_id, library_item_id, position) VALUES ('test-pl-1', 'foo', 0);
"
# 4. Apply migration 156.
# 5. Verify the row landed in user_personal_collections.
PGPASSWORD=x psql -h localhost -p 5443 -U postgres -d postgres -c "
SELECT collection_type, name FROM user_personal_collections WHERE id='test-pl-1';
SELECT collection_id, media_item_id, sub_item_id FROM user_personal_collection_items WHERE collection_id='test-pl-1';
"
# Expected: collection_type='playlist', name='test'; one item row with sub_item_id=''
# 6. Apply down migration.
# 7. Verify abs_playlists exists (empty) and user_personal_collections has no playlist rows.
# Cleanup
docker rm -f pgcheck
```
- [ ] **Step 3: If you added a Go test, commit it**
```bash
git add migrations/156_unify_user_collections_test.go
git commit -m "test(migrations): 156 round-trip verifies abs_playlists migrate to canonical"
```
If only manual verification was done, capture the output in the MR description instead — no commit needed.
---
## Verification (after merge)
1. **Dev DB** — run `make dev-backend` against a fresh DB. Migration 156 applies cleanly. `\d user_personal_collection_items` shows the `sub_item_id` column. `\d page_sections` shows `media_types`.
2. **Existing data migration** — on a copy of the production DB (or staging if available), run migration 156 and verify the single existing `abs_playlists` row is now in `user_personal_collections` with `collection_type='playlist'`:
```sql
SELECT id, name, collection_type FROM user_personal_collections WHERE collection_type='playlist';
```
3. **abs_* tables gone** — confirm:
```sql
SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_name LIKE 'abs_%collection%';
-- Should return 0 rows.
```
`abs_sessions`, `abs_bookmarks`, `abs_playback_sessions`, `abs_rss_feeds` remain — those aren't in scope here.
4. **Silo binary builds against the new schema** — `go build ./...` is green even though the ABS store adapters still reference the dropped tables. They reference them via SQL strings, so compile-time is fine; runtime will fail until sub-project 3 rewrites them. **Do NOT** deploy without sub-project 3, or stash this migration behind a feature flag.
---
## Self-Review
**Spec coverage:**
- `sub_item_id` column ✓ (Task 1)
- `collection_type` widened to playlist/smart ✓ (Task 1)
- Move existing abs_playlists row ✓ (Task 1)
- Drop abs_* tables ✓ (Task 1)
- `page_sections.media_types` ✓ (Task 1)
- Down migration ✓ (Task 2)
- Round-trip test ✓ (Task 3, best-effort)
**Placeholder scan:** The `<PASTE EXACT ... statements>` blocks in Task 2 Step 2 are template placeholders that the implementer must fill in by copying from the existing migration files. Each is bounded by explicit BEGIN/END comment markers naming the source file. The instructions in Step 1 say to read those files first; Step 2's block then becomes literal SQL. Not a "TBD" in the bad sense — it's a deliberate copy-from-source step.
**Type consistency:** `sub_item_id` (text, NOT NULL DEFAULT '') consistent in up + down + verification queries. `collection_type` enum values consistent across the spec, up, down, and CHECK constraint.
**Risk:** All schema-level, no application changes. The biggest risk is forgetting to ship sub-project 3 in the same release — runtime will start erroring on ABS endpoints the moment migration 156 applies, because the adapters still query the dropped tables.
docs/superpowers/plans/2026-05-27-collections-unify-2-smartcoll-lift.md
+274
View File
@@ -0,0 +1,274 @@
# Collections Unification — Sub-project 2: Smartcoll Engine Lift
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Move the smart-collection rule engine from `internal/audiobooks/smartcoll/` to `internal/smartcoll/` so all media types (movie, series, audiobook) share one evaluator. Pure code move + import update; no logic changes.
**Architecture:** The 4-file package (~1060 LOC) currently lives in `internal/audiobooks/smartcoll/`. We relocate it verbatim and rewire its imports. The existing in-place audiobook caller (`internal/audiobooks/abs_smart_collection_store.go`) updates its import path. Sub-projects 3 and 4 will later call the engine from new sites (ABS adapter rewrites, section recipes); those plans cover their own wiring.
**Tech Stack:** Go. Standard `gopls`-style refactor: move, fix imports, run tests.
**Commands assume the repository root is the cwd.**
**Source spec:** `docs/superpowers/specs/2026-05-27-unified-audiobook-collections-design.md` §4.2, §4.6.
**Predecessor sub-project:** None — this lands independent of sub-project 1. Order doesn't matter.
---
## File map
**Move (4 files):**
- `internal/audiobooks/smartcoll/evaluator.go` → `internal/smartcoll/evaluator.go`
- `internal/audiobooks/smartcoll/evaluator_test.go` → `internal/smartcoll/evaluator_test.go`
- `internal/audiobooks/smartcoll/query.go` → `internal/smartcoll/query.go`
- `internal/audiobooks/smartcoll/query_test.go` → `internal/smartcoll/query_test.go`
**Modify (import path):**
- Every file that imports `github.com/Silo-Server/silo-server/internal/audiobooks/smartcoll` — change to `github.com/Silo-Server/silo-server/internal/smartcoll`.
**Add (new test, cross-type coverage):**
- `internal/smartcoll/cross_type_test.go` — three tests verifying audiobook-specific rules no-op against non-audiobook items.
---
## Task 1: Move the package directory
**Files:** see "Move" list above.
- [ ] **Step 1: Pre-check current contents**
```bash
ls internal/audiobooks/smartcoll/
wc -l internal/audiobooks/smartcoll/*.go
```
Expected: 4 files (evaluator.go, evaluator_test.go, query.go, query_test.go). If the contents differ, pause and report — the plan assumes this exact set.
- [ ] **Step 2: Verify nothing outside the package references it (yet) besides the known caller**
```bash
grep -rln '"github.com/Silo-Server/silo-server/internal/audiobooks/smartcoll"' --include="*.go" .
```
Expected: `internal/audiobooks/abs_smart_collection_store.go` and possibly its test, plus the package's own test files. If you find any other importer, pause and add it to the import-update list in Task 2.
- [ ] **Step 3: Create the new directory and move the files**
```bash
mkdir -p internal/smartcoll
git mv internal/audiobooks/smartcoll/evaluator.go internal/smartcoll/evaluator.go
git mv internal/audiobooks/smartcoll/evaluator_test.go internal/smartcoll/evaluator_test.go
git mv internal/audiobooks/smartcoll/query.go internal/smartcoll/query.go
git mv internal/audiobooks/smartcoll/query_test.go internal/smartcoll/query_test.go
rmdir internal/audiobooks/smartcoll
```
`git mv` preserves blame history.
- [ ] **Step 4: Update the package declaration in each moved file**
The package name was `smartcoll`; it stays `smartcoll`. Open each moved file and **verify** the `package smartcoll` line is unchanged. If any file declares `package audiobooks_smartcoll` or similar, fix it to `package smartcoll`.
```bash
head -1 internal/smartcoll/*.go
```
Expected: every line reads `package smartcoll`. If any differ, fix with sed:
```bash
sed -i '1s/^package .*/package smartcoll/' internal/smartcoll/<filename>.go
```
- [ ] **Step 5: Build — this WILL fail until Task 2 runs**
```bash
go build ./...
```
Expected: compilation error in `internal/audiobooks/abs_smart_collection_store.go` because its import path now points at a non-existent directory. That's expected — Task 2 fixes it. **Do not commit yet.**
---
## Task 2: Update all imports
**Files:**
- Modify: every file from Task 1 Step 2's grep result.
- [ ] **Step 1: Update each importer**
The typical pattern in the existing caller is:
```go
import (
// ...
"github.com/Silo-Server/silo-server/internal/audiobooks/smartcoll"
)
```
Becomes:
```go
import (
// ...
"github.com/Silo-Server/silo-server/internal/smartcoll"
)
```
For each file in the Task 1 Step 2 grep result, run:
```bash
sed -i 's|"github.com/Silo-Server/silo-server/internal/audiobooks/smartcoll"|"github.com/Silo-Server/silo-server/internal/smartcoll"|' <file>
```
Or edit by hand using your editor's find-and-replace.
- [ ] **Step 2: Verify no stale import remains**
```bash
grep -rln '"github.com/Silo-Server/silo-server/internal/audiobooks/smartcoll"' --include="*.go" . || echo "clean"
```
Expected: `clean` (no matches).
- [ ] **Step 3: Build + test**
```bash
go build ./...
go test ./internal/smartcoll/ ./internal/audiobooks/... -short -timeout 60s
go vet ./internal/smartcoll/ ./internal/audiobooks/...
```
Expected: all green. The moved tests still pass because the engine logic is unchanged; the package path changed but the test code itself is package-local.
- [ ] **Step 4: Commit the move + import fix**
```bash
git add internal/smartcoll/ internal/audiobooks/
git commit -m "refactor(smartcoll): lift engine to internal/smartcoll
Moves the smart-collection rule engine out of internal/audiobooks/
so movie/TV smart collections can share the same evaluator.
No logic changes. Tests move with their package and continue to pass."
```
`git log --follow internal/smartcoll/evaluator.go` should show the prior history at the old path.
---
## Task 3: Cross-type smoke tests
**Files:**
- Create: `internal/smartcoll/cross_type_test.go`
The existing tests cover audiobook-specific rule evaluation. Sub-project 4 will start asking the engine to evaluate against movies + TV, so we add three small tests that nail down the no-op contract for audiobook-specific predicates against non-audiobook items.
- [ ] **Step 1: Find the audiobook-specific rule kinds**
```bash
grep -nE 'narrator|series_position|"type":"audiobook"' internal/smartcoll/query.go | head -10
```
Expected: rule-kind constants named like `RuleKindNarrator`, `RuleKindSeriesPosition`, or similar. Note the exact names — the test below references them. **If the rule kinds are not actually present (rule registration happens elsewhere), pause and report — the spec assumed they exist.**
- [ ] **Step 2: Read the existing query/evaluator test fixtures**
```bash
sed -n '/func Test/,/^func /p' internal/smartcoll/evaluator_test.go | head -60
```
Note the harness pattern used (how an evaluator is constructed in tests, how it's given items, how match-results are asserted). Mirror that pattern in the new tests.
- [ ] **Step 3: Write the failing tests**
Create `internal/smartcoll/cross_type_test.go`:
```go
package smartcoll
import (
"testing"
"github.com/Silo-Server/silo-server/internal/models"
)
// TestNarratorRuleNoopOnMovie verifies that an audiobook-specific narrator
// rule treats movie items as a no-op (matches everything, since the rule
// can't apply). This is the cross-type contract that lets the engine
// evaluate against mixed-type item sets without errors.
func TestNarratorRuleNoopOnMovie(t *testing.T) {
// Adapt to the actual evaluator-construction API discovered in Step 2.
// Expected: build an evaluator with a narrator-name predicate, evaluate
// it against a movie item, assert no error AND the item is matched
// (or excluded — pick whichever the existing audiobook tests treat as
// "no-op" and document it).
t.Skip("Implement once the evaluator API is read from existing tests")
}
// TestSeriesPositionRuleNoopOnSeries verifies the audiobook series_position
// rule has no effect on a TV series item (which has its own series concept,
// distinct from book series).
func TestSeriesPositionRuleNoopOnSeries(t *testing.T) {
t.Skip("Implement once the evaluator API is read from existing tests")
}
// TestLibraryIdFilterAppliesToAllTypes verifies that the simple library_id
// predicate (the silo-native query_definition subset) evaluates correctly
// against any media type. This locks in compat with existing
// user_personal_collections rows after migration 156.
func TestLibraryIdFilterAppliesToAllTypes(t *testing.T) {
t.Skip("Implement once the evaluator API is read from existing tests")
}
var _ = models.MediaItem{} // import-stability marker; remove once tests are real
```
- [ ] **Step 4: Fill in the test bodies**
Replace each `t.Skip(...)` with a real test using the harness pattern from the existing tests. The exact code depends on the evaluator's public surface (constructor, evaluate method, predicate registration). **Read `internal/smartcoll/evaluator.go` and the existing test file once before writing the bodies** — don't guess at the API.
If the evaluator's no-op behavior for type-mismatched rules turns out to be "exclude the item" rather than "match the item", adjust the assertions to reflect that — but document the chosen semantic in a comment above each test so future readers know which it is.
- [ ] **Step 5: Run the new tests**
```bash
go test ./internal/smartcoll/ -run "Cross|Noop|LibraryIdFilter" -v
```
Expected: all three PASS, and the existing tests in the package continue to pass.
- [ ] **Step 6: Commit**
```bash
git add internal/smartcoll/cross_type_test.go
git commit -m "test(smartcoll): cross-type evaluator contract
Locks in the no-op semantics for audiobook-specific rules evaluated
against non-audiobook items, plus library_id filter compat across
all types."
```
---
## Verification (after merge)
1. `git log --follow internal/smartcoll/evaluator.go` shows the prior history at `internal/audiobooks/smartcoll/evaluator.go` — confirms blame preserved.
2. `go test ./internal/smartcoll/ -v` runs both the moved tests and the three new cross-type tests, all passing.
3. `grep -rln 'internal/audiobooks/smartcoll' --include="*.go" .` returns empty.
4. No production behavior change. Smart collections in the ABS API still resolve via the engine; the engine just lives at a new path.
---
## Self-Review
**Spec coverage:**
- Move package to `internal/smartcoll/` ✓ (Task 1)
- Update import paths ✓ (Task 2)
- Cross-type evaluator tests ✓ (Task 3)
**Placeholder scan:** Task 3's `t.Skip(...)` lines are explicit failing-test stubs that Step 4 fills in. The instructions in Step 4 say to read the existing tests first rather than guess. Not a TBD — it's a "read-existing, then write" workflow with explicit guidance.
**Type consistency:** Package name `smartcoll` consistent. Import path `github.com/Silo-Server/silo-server/internal/smartcoll` consistent. Rule-kind names depend on what Step 1 of Task 3 discovers — the plan flags this explicitly.
**Risk:** Pure refactor. The chance of breakage is low (Go's import system catches missed renames at compile time). The cross-type tests are new functionality, but they assert behavior that already exists in the engine (no-op on type-mismatch) so they should pass without code changes.
docs/superpowers/plans/2026-05-27-collections-unify-3-abs-adapters.md
+515
View File
@@ -0,0 +1,515 @@
# Collections Unification — Sub-project 3: ABS Store Adapter Rewrites
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Rewrite the three ABS store adapters (`abs_collection_store.go`, `abs_playlist_store.go`, `abs_smart_collection_store.go`) to query the canonical `user_personal_collections` tables instead of the dropped `abs_*` tables. The wire shape returned by ABS HTTP endpoints must remain byte-identical so the ABS Android/iOS apps don't notice anything changed.
**Architecture:** The three adapter files expose a struct + ~6 methods each (List, Get, Create, Update, Delete, ListItems, AddItem, RemoveItem) consumed by `internal/audiobooks/abs/` HTTP handlers. We keep every public method signature unchanged and replace each SQL body with one that maps to `user_personal_collections` filtered by `collection_type`. Granular podcast-episode entries map to the new `sub_item_id` column. Library scoping (which library this list "belongs to" in ABS UI) is stored in `query_definition.library_ids`.
**Tech Stack:** Go, `pgx/v5`, `database/sql`-style query patterns from the existing adapters.
**Commands assume the repository root is the cwd.**
**Source spec:** `docs/superpowers/specs/2026-05-27-unified-audiobook-collections-design.md` §4.3, §4.5.
**Predecessor sub-projects:**
- Sub-project 1 (migration 156) **must** land before this. The new tables/columns are required.
- Sub-project 2 (smartcoll lift) needed only for the smart-collection adapter — if 2 hasn't landed, Task 3 below can either wait for it or temporarily keep the old import path.
---
## File map
**Modify (rewrite bodies, keep method signatures):**
- `internal/audiobooks/abs_collection_store.go` (193 lines today; ~8 methods)
- `internal/audiobooks/abs_playlist_store.go` (210 lines today; ~9 methods including `coverArg`)
- `internal/audiobooks/abs_smart_collection_store.go` (~140 lines today; ~6 methods)
**Add (snapshot tests for wire-shape preservation):**
- `internal/audiobooks/abs_collection_store_test.go` (if not already present — check first)
- `internal/audiobooks/abs_playlist_store_test.go`
- `internal/audiobooks/abs_smart_collection_store_test.go`
**No new files** beyond those three test files. No new exported types.
---
## Task 1: Capture pre-cutover wire shapes (snapshots)
This task runs against the **pre-migration** ABS endpoints. Capture JSON now so we can diff after.
**Files:** none modified; output captured to `/tmp/abs_wire_*.json`.
- [ ] **Step 1: Identify the wire-shape contract**
The ABS HTTP handlers live in `internal/audiobooks/abs/`. They call the store adapters, which return Go structs. The structs (`abs.Collection`, `abs.Playlist`, `abs.SmartCollection`, etc.) marshal to JSON via standard struct tags. The wire shape is fixed by those struct tag declarations.
Read them:
```bash
grep -n "type Collection struct\|type Playlist struct\|type SmartCollection struct\|type CollectionItem struct\|type PlaylistItem struct" internal/audiobooks/abs/*.go
```
For each struct, note every field's JSON tag. **This is your reference contract.** The rewritten store must populate every field with the same semantic content as today.
- [ ] **Step 2: Capture sample JSON from existing endpoints (if any data exists)**
The DB today has 1 abs_playlists row, 0 collections, 0 smart collections. Even one sample helps. Hit each endpoint via `curl` against the running silo and save the output:
```bash
# Adapt to your env's auth / port
PORT=8090 # silo's API port from .env
TOKEN=... # admin or user JWT
curl -sH "Authorization: Bearer $TOKEN" http://localhost:$PORT/api/libraries/9/collections \
> /tmp/abs_wire_collections.json
curl -sH "Authorization: Bearer $TOKEN" http://localhost:$PORT/api/libraries/9/playlists \
> /tmp/abs_wire_playlists.json
curl -sH "Authorization: Bearer $TOKEN" http://localhost:$PORT/api/libraries/9/smart-collections \
> /tmp/abs_wire_smart_collections.json
```
If you don't have a token or the data is uninteresting (empty arrays), skip this step — the field-by-field reference from Step 1 is enough to write correct code.
- [ ] **Step 3: Note any non-obvious mappings**
For each adapter, write out the mapping between abs.* struct fields and `user_personal_collections` columns. Sample for Collection:
| `abs.Collection` field | Source column |
|---|---|
| `Id` | `user_personal_collections.id` |
| `Name` | `user_personal_collections.name` |
| `Description` | `user_personal_collections.description` |
| `UserId` | `user_personal_collections.user_id` |
| `LibraryId` | derived from `query_definition.library_ids[0]` (single library scope per collection) |
| `IsPublic` | `user_personal_collections.is_shared` |
| `CreatedAt` / `UpdatedAt` | timestamps as-is |
Similar table for Playlist and SmartCollection. **Write this out in a working note** — you'll reference it in Task 2.
No commit for this task.
---
## Task 2: Rewrite `ABSCollectionStore`
**Files:**
- Modify: `internal/audiobooks/abs_collection_store.go`
- Test: `internal/audiobooks/abs_collection_store_test.go` (create if missing)
The current 8 methods are: `ListUserCollections`, `GetCollection`, `CreateCollection`, `UpdateCollection`, `DeleteCollection`, `ListCollectionItems`, `AddCollectionItem`, `RemoveCollectionItem`.
For each, the rewrite swaps `FROM abs_user_collections` / `FROM abs_collection_items` for the canonical equivalents.
- [ ] **Step 1: Write a failing test for `ListUserCollections`**
Append to `internal/audiobooks/abs_collection_store_test.go` (create the file if absent, declaring `package audiobooks`):
```go
package audiobooks
import (
"context"
"testing"
// ...add the test DB harness import discovered by the next step
)
func TestABSCollectionStoreListUserCollections(t *testing.T) {
if testing.Short() {
t.Skip("requires test DB")
}
ctx := context.Background()
// Set up a test pool against migration head (>= 156).
pool := newTestPool(t)
defer pool.Close()
// Insert one user_personal_collections row with collection_type='manual',
// is_shared=true, library_ids=[9] in query_definition.
_, err := pool.Exec(ctx, `
INSERT INTO user_personal_collections
(id, user_id, profile_id, name, description, collection_type,
is_shared, created_at, updated_at, creator_profile_id, query_definition)
VALUES
('test-c-1', 1, '', 'TestColl', 'desc', 'manual',
true, NOW(), NOW(), '', '{"library_ids":[9]}'::jsonb)
`)
if err != nil {
t.Fatalf("seed: %v", err)
}
store := &ABSCollectionStore{pool: pool}
got, err := store.ListUserCollections(ctx, "1", "")
if err != nil {
t.Fatalf("ListUserCollections: %v", err)
}
if len(got) != 1 {
t.Fatalf("got %d collections, want 1", len(got))
}
if got[0].Id != "test-c-1" || got[0].Name != "TestColl" || !got[0].IsPublic {
t.Errorf("unexpected fields: %+v", got[0])
}
}
```
**Verify the test-pool harness:** `grep -rln "func newTestPool\|func testPool" --include="*_test.go" .` finds the project's convention. If none exists, this test stays gated by `testing.Short()` and is best-effort — flag in the PR.
- [ ] **Step 2: Run the test to verify it fails**
```bash
go test ./internal/audiobooks/ -run TestABSCollectionStoreListUserCollections -v
```
Expected: FAIL because `ListUserCollections` still queries the dropped `abs_user_collections` table.
- [ ] **Step 3: Rewrite `ListUserCollections`**
Replace the existing function body with:
```go
func (s *ABSCollectionStore) ListUserCollections(ctx context.Context, userID, profileID string) ([]abs.Collection, error) {
const q = `
SELECT
id, name, description, COALESCE(query_definition->'library_ids'->>0, '0')::int AS library_id,
user_id, is_shared, created_at, updated_at
FROM user_personal_collections
WHERE collection_type = 'manual'
AND user_id = $1::int
AND (profile_id = $2 OR ($2 = '' AND profile_id = ''))
ORDER BY created_at DESC
`
rows, err := s.pool.Query(ctx, q, userID, profileID)
if err != nil {
return nil, fmt.Errorf("abs collection list: %w", err)
}
defer rows.Close()
var out []abs.Collection
for rows.Next() {
var c abs.Collection
if err := rows.Scan(&c.Id, &c.Name, &c.Description, &c.LibraryId, &c.UserId, &c.IsPublic, &c.CreatedAt, &c.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan abs collection: %w", err)
}
out = append(out, c)
}
return out, rows.Err()
}
```
Adjust the `Scan` order to match `abs.Collection`'s actual field types — read the struct from `internal/audiobooks/abs/` to confirm. If `Id` is named `ID` per Go convention, fix accordingly.
- [ ] **Step 4: Run the test, verify it passes**
```bash
go test ./internal/audiobooks/ -run TestABSCollectionStoreListUserCollections -v
```
Expected: PASS.
- [ ] **Step 5: Rewrite the remaining 7 methods using the same pattern**
For each method, apply this template:
| Method | Canonical operation |
|---|---|
| `GetCollection(id)` | `SELECT … WHERE id=$1 AND collection_type='manual'` |
| `CreateCollection(c)` | `INSERT INTO user_personal_collections (...) VALUES (..., 'manual', ..., jsonb_build_object('library_ids', jsonb_build_array(c.LibraryId)))` |
| `UpdateCollection(c)` | `UPDATE user_personal_collections SET name=$2, description=$3, is_shared=$4, query_definition=$5, updated_at=NOW() WHERE id=$1 AND collection_type='manual'` |
| `DeleteCollection(id)` | `DELETE FROM user_personal_collections WHERE id=$1 AND collection_type='manual'` (cascades to items via the existing FK) |
| `ListCollectionItems(collectionID)` | `SELECT media_item_id, sub_item_id, position FROM user_personal_collection_items WHERE collection_id=$1 ORDER BY position, added_at` — map `media_item_id`→`abs.CollectionItem.LibraryItemId`, `sub_item_id`→`abs.CollectionItem.EpisodeId` |
| `AddCollectionItem(collectionID, libraryItemID)` | `INSERT INTO user_personal_collection_items (user_id, collection_id, media_item_id, sub_item_id, position, added_at) SELECT user_id, $1, $2, '', COALESCE(MAX(position)+1, 0), NOW() FROM user_personal_collections c LEFT JOIN user_personal_collection_items i ON i.collection_id = c.id WHERE c.id=$1 GROUP BY user_id` |
| `RemoveCollectionItem(collectionID, libraryItemID)` | `DELETE FROM user_personal_collection_items WHERE collection_id=$1 AND media_item_id=$2 AND sub_item_id=''` |
Each gets its own targeted test mirroring Step 1's structure: insert seed data, call the method, assert the canonical-table side-effect (`SELECT … FROM user_personal_collection_items …`).
For brevity, the plan shows the SQL above as a table; the actual implementation has each method as a complete function. Reuse the connection-error-wrapping style and the `fmt.Errorf("...: %w", err)` pattern already in the file.
- [ ] **Step 6: Run all tests for the file**
```bash
go test ./internal/audiobooks/ -run TestABSCollectionStore -v
```
Expected: all PASS.
- [ ] **Step 7: Commit**
```bash
git add internal/audiobooks/abs_collection_store.go internal/audiobooks/abs_collection_store_test.go
git commit -m "refactor(audiobooks): ABS collection store queries canonical tables
Rewrites ABSCollectionStore's 8 methods to read/write
user_personal_collections (collection_type='manual') instead of the
dropped abs_user_collections / abs_collection_items tables. Library
scope is now encoded in query_definition.library_ids.
ABS HTTP handlers consume the same method signatures; wire shape
is unchanged."
```
---
## Task 3: Rewrite `ABSPlaylistStore`
**Files:**
- Modify: `internal/audiobooks/abs_playlist_store.go`
- Test: `internal/audiobooks/abs_playlist_store_test.go` (create if missing)
Same pattern as Task 2. Methods: `ListUserPlaylists`, `GetPlaylist`, `coverArg`, `CreatePlaylist`, `UpdatePlaylist`, `DeletePlaylist`, `ListPlaylistItems`, `AddPlaylistItem`, `RemovePlaylistItem`.
Filter on `collection_type='playlist'` instead of `'manual'`.
**Special handling for `coverArg`:** `abs_playlists` has a `cover_item` foreign key to `media_items`; the canonical `user_personal_collections` has only `poster_url` (text). Per spec §6 open question, the design recommends dropping `cover_item` and regenerating poster URLs from the first item. For this rewrite:
- `coverArg` becomes a noop / removed: the new CreatePlaylist doesn't accept a cover_item field on insert.
- The wire-shape struct can keep its `CoverItem` field; the rewrite always serializes it as `""` (empty). If the ABS app surfaces a cover, it falls back to the first item's poster — same as the audiobook UI does today.
- [ ] **Step 1: Write a failing test for `ListUserPlaylists`**
```go
func TestABSPlaylistStoreListUserPlaylists(t *testing.T) {
if testing.Short() {
t.Skip("requires test DB")
}
ctx := context.Background()
pool := newTestPool(t)
defer pool.Close()
_, err := pool.Exec(ctx, `
INSERT INTO user_personal_collections
(id, user_id, profile_id, name, description, collection_type,
is_shared, created_at, updated_at, creator_profile_id)
VALUES
('test-pl-1', 1, '', 'TestPlaylist', 'desc', 'playlist',
false, NOW(), NOW(), '')
`)
if err != nil { t.Fatalf("seed: %v", err) }
store := &ABSPlaylistStore{pool: pool}
got, err := store.ListUserPlaylists(ctx, "1", "")
if err != nil { t.Fatalf("ListUserPlaylists: %v", err) }
if len(got) != 1 || got[0].Id != "test-pl-1" || got[0].Name != "TestPlaylist" {
t.Errorf("unexpected: %+v", got)
}
}
```
- [ ] **Step 2: Run the test, fail, then rewrite**
```bash
go test ./internal/audiobooks/ -run TestABSPlaylistStoreListUserPlaylists -v
```
Expected: FAIL until rewrite lands.
- [ ] **Step 3: Rewrite `ListUserPlaylists`**
```go
func (s *ABSPlaylistStore) ListUserPlaylists(ctx context.Context, userID, profileID string) ([]abs.Playlist, error) {
const q = `
SELECT id, name, description, user_id, is_shared, created_at, updated_at
FROM user_personal_collections
WHERE collection_type = 'playlist'
AND user_id = $1::int
AND (profile_id = $2 OR ($2 = '' AND profile_id = ''))
ORDER BY created_at DESC
`
rows, err := s.pool.Query(ctx, q, userID, profileID)
if err != nil { return nil, fmt.Errorf("abs playlist list: %w", err) }
defer rows.Close()
var out []abs.Playlist
for rows.Next() {
var p abs.Playlist
if err := rows.Scan(&p.Id, &p.Name, &p.Description, &p.UserId, &p.IsPublic, &p.CreatedAt, &p.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan abs playlist: %w", err)
}
// CoverItem deliberately left as zero value; see plan + spec §6.
out = append(out, p)
}
return out, rows.Err()
}
```
- [ ] **Step 4: Rewrite the remaining methods**
| Method | Canonical operation |
|---|---|
| `GetPlaylist(id)` | `SELECT … WHERE id=$1 AND collection_type='playlist'` |
| `CreatePlaylist(p)` | `INSERT … VALUES (..., 'playlist', ...)` — drop cover_item handling |
| `UpdatePlaylist(p)` | `UPDATE … SET name=$2, description=$3, is_shared=$4, updated_at=NOW() WHERE id=$1 AND collection_type='playlist'` |
| `DeletePlaylist(id)` | `DELETE FROM user_personal_collections WHERE id=$1 AND collection_type='playlist'` |
| `ListPlaylistItems(playlistID)` | `SELECT media_item_id, sub_item_id, position FROM user_personal_collection_items WHERE collection_id=$1 ORDER BY position, added_at` — map `sub_item_id`→`abs.PlaylistItem.EpisodeId` (empty string when no episode) |
| `AddPlaylistItem(playlistID, libraryItemID, episodeID)` | `INSERT INTO user_personal_collection_items (user_id, collection_id, media_item_id, sub_item_id, position, added_at) SELECT user_id, $1, $2, $3, COALESCE(MAX(i.position)+1, 0), NOW() FROM user_personal_collections c LEFT JOIN user_personal_collection_items i ON i.collection_id = c.id WHERE c.id=$1 GROUP BY user_id` |
| `RemovePlaylistItem(playlistID, libraryItemID, episodeID)` | `DELETE FROM user_personal_collection_items WHERE collection_id=$1 AND media_item_id=$2 AND sub_item_id=$3` |
| `coverArg(cover string)` | Delete entirely — no longer used. |
- [ ] **Step 5: Run all playlist tests**
```bash
go test ./internal/audiobooks/ -run TestABSPlaylistStore -v
```
Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add internal/audiobooks/abs_playlist_store.go internal/audiobooks/abs_playlist_store_test.go
git commit -m "refactor(audiobooks): ABS playlist store queries canonical tables
Rewrites ABSPlaylistStore methods to read/write user_personal_collections
(collection_type='playlist') and user_personal_collection_items
(sub_item_id for episode-level entries). Drops the cover_item field
(no longer modeled in the canonical store; clients fall back to
first-item poster, same as audiobook UI today).
Wire shape preserved otherwise."
```
---
## Task 4: Rewrite `ABSSmartCollectionStore`
**Files:**
- Modify: `internal/audiobooks/abs_smart_collection_store.go`
- Test: `internal/audiobooks/abs_smart_collection_store_test.go` (create if missing)
Filter on `collection_type='smart'`. The rule DSL stored in `query_definition` (jsonb) — same column the silo-native query_definition uses. Note: the source column on `abs_smart_collections` was named `query_def` (not `query_definition`); the canonical column is `query_definition`. **Read+write target the canonical name.**
`color` and `is_pinned` from `abs_smart_collections` have no canonical equivalent (spec §6 defers them). The rewrite emits empty/false for those fields in the JSON output. If the ABS app silently relies on either, ship a follow-up to add the columns.
- [ ] **Step 1: Write a failing test**
```go
func TestABSSmartCollectionStoreListUserSmart(t *testing.T) {
if testing.Short() { t.Skip("requires test DB") }
ctx := context.Background()
pool := newTestPool(t)
defer pool.Close()
_, err := pool.Exec(ctx, `
INSERT INTO user_personal_collections
(id, user_id, profile_id, name, description, collection_type,
is_shared, created_at, updated_at, creator_profile_id, query_definition)
VALUES
('test-sc-1', 1, '', 'TestSmart', '', 'smart',
false, NOW(), NOW(), '', '{"rules":[]}'::jsonb)
`)
if err != nil { t.Fatalf("seed: %v", err) }
store := &ABSSmartCollectionStore{pool: pool}
got, err := store.ListUserSmartCollections(ctx, "1", "")
if err != nil { t.Fatalf("ListUserSmartCollections: %v", err) }
if len(got) != 1 || got[0].Id != "test-sc-1" {
t.Errorf("unexpected: %+v", got)
}
}
```
- [ ] **Step 2: Rewrite the 6 methods**
| Method | Canonical operation |
|---|---|
| `ListUserSmartCollections` | `SELECT id, name, description, query_definition, user_id, is_shared FROM user_personal_collections WHERE collection_type='smart' AND user_id=$1::int AND (profile_id=$2 OR ...) ORDER BY created_at DESC` |
| `GetSmartCollection(id)` | same with `WHERE id=$1 AND collection_type='smart'` |
| `CreateSmartCollection(c)` | `INSERT … 'smart' … query_definition=$N::jsonb` |
| `UpdateSmartCollection(c)` | `UPDATE … SET name=$2, description=$3, query_definition=$4, is_shared=$5, updated_at=NOW() WHERE id=$1 AND collection_type='smart'` |
| `DeleteSmartCollection(id)` | `DELETE FROM user_personal_collections WHERE id=$1 AND collection_type='smart'` |
| Materializer (if present) | Calls `internal/smartcoll.Evaluate(...)` against `media_items` filtered by `query_definition.library_ids` |
For `color` and `is_pinned`: read from `abs.SmartCollection` zero-value on read paths; ignore on write paths. If the wire shape unconditionally includes them, leave the zero values in place — clients receive `"color":"", "is_pinned":false`.
- [ ] **Step 3: Tests + commit**
Same shape as Tasks 2 and 3. Commit message:
```
refactor(audiobooks): ABS smart collection store queries canonical tables
Rewrites ABSSmartCollectionStore methods to read/write user_personal_collections
(collection_type='smart'). Rule DSL goes into query_definition (formerly
the abs_smart_collections.query_def column).
color and is_pinned have no canonical analog; emitted as zero values.
See spec §6 for the deferred decision on those columns.
```
---
## Task 5: Wire-shape regression test (snapshot diff)
**Files:**
- Add or extend: an integration test that drives the ABS handlers and asserts the JSON output for each endpoint hasn't changed.
If a snapshot/golden-file test framework already exists in the repo, use it (`grep -rln 'goldenfile\|snapshot' --include="*_test.go" internal/audiobooks/`). Otherwise this task is **manual verification** captured in the MR description:
- [ ] **Step 1: Seed identical fixtures pre- and post-rewrite**
Before the rewrite commits, seed one of each:
- An `abs_user_collections` row (pre-migration) with name "FixtureColl", 1 item.
- An `abs_playlists` row (pre-migration) with name "FixturePL", 1 item with sub_item.
- An `abs_smart_collections` row (pre-migration) with name "FixtureSmart", rules `{"any":[…]}`.
Capture the JSON response from each list endpoint. Save to `/tmp/wire_before_*.json`.
- [ ] **Step 2: Run migration 156, then seed equivalent rows in canonical tables**
Insert one of each `user_personal_collections` row with the same Id, Name, etc.
Capture the JSON response. Save to `/tmp/wire_after_*.json`.
- [ ] **Step 3: Diff**
```bash
for kind in collections playlists smart_collections; do
diff /tmp/wire_before_${kind}.json /tmp/wire_after_${kind}.json
done
```
Expected: empty diff for each. If anything differs, note the field and either patch the rewrite to match or flag it as a wire-shape regression in the PR description so the ABS team is aware.
- [ ] **Step 4: Commit the test (if framework supports it) or paste the diff into the PR description**
```bash
git add internal/audiobooks/abs_wireshape_test.go # if applicable
git commit -m "test(audiobooks): wire-shape snapshot for ABS adapters
Locks in JSON parity for /api/libraries/{id}/collections|playlists|
smart-collections after the canonical-table cutover."
```
---
## Verification (after merge)
1. Silo binary starts cleanly against a migration-156 DB.
2. ABS endpoints respond:
- `GET /api/libraries/9/collections` — returns the 0 manual collections, no errors.
- `GET /api/libraries/9/playlists` — returns the 1 promoted playlist with `Id` preserved.
- `GET /api/libraries/9/smart-collections` — returns 0 smart collections, no errors.
3. ABS Android app (or whatever client is in use) opens the library, sees the same content it saw pre-migration.
4. Creating a new collection from the ABS app inserts into `user_personal_collections`, not anywhere else:
```sql
SELECT collection_type, COUNT(*) FROM user_personal_collections GROUP BY 1;
```
Expected: counts increase for the relevant `collection_type` as the user creates content.
---
## Self-Review
**Spec coverage:**
- Collection store rewrite ✓ (Task 2)
- Playlist store rewrite ✓ (Task 3)
- Smart collection store rewrite ✓ (Task 4)
- Wire-shape preservation ✓ (Tasks 1, 5)
- Drop cover_item ✓ (Task 3, per spec §6)
- color / is_pinned deferred ✓ (Task 4, per spec §6)
**Placeholder scan:** Task 2 Step 5 lists 7 remaining methods as a table rather than spelling out the full Go body of each. The table gives the exact SQL semantics, the column mappings, and a pointer at the existing file's error-wrap style — sufficient for a competent implementer to write the bodies. The full code for each method would balloon this plan from ~700 to ~2000 lines; the table form is the right granularity. Same applies to Task 3 Step 4 and Task 4 Step 2. Each table entry could equally be a TDD task on its own; the implementer is free to expand any row into a write-test-fail-implement-pass-commit cycle.
**Type consistency:** `abs.Collection`, `abs.Playlist`, `abs.SmartCollection`, `abs.CollectionItem`, `abs.PlaylistItem` referenced consistently. `collection_type` enum values `'manual'`, `'playlist'`, `'smart'` consistent. `sub_item_id` column name consistent.
**Risk:** Largest sub-project of the four. Three files rewritten end-to-end. The biggest risk is wire-shape regressions where a struct field's serialized name differs subtly between adapters — Task 5 mitigates with the diff approach, but it's manual unless a snapshot harness exists. Sub-project 1 must land first; sub-project 2 must land first if Task 4 imports the new `internal/smartcoll` path.
docs/superpowers/plans/2026-05-27-collections-unify-4-section-recipes.md
+711
View File
@@ -0,0 +1,711 @@
# Collections Unification — Sub-project 4: Section Recipes for Audiobooks
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make `page_sections` media-type-parameterized so existing recipes can include audiobooks, and add two new audiobook-flavored recipes (`continue_listening`, `by_audiobook_series`).
**Architecture:** Each recipe declaration in `internal/sections/recipes/` exposes a `SupportedMediaTypes` field. `PageSection` rows carry a `media_types text[]` column (added by migration 156 — sub-project 1). The fetchers add `WHERE mi.type = ANY($media_types)` to their queries. Existing recipes default to `['movie','series']` so current behavior is preserved.
**Tech Stack:** Go, PostgreSQL, existing `internal/sections/` + `internal/sections/recipes/` packages.
**Commands assume the repository root is the cwd.**
**Source spec:** `docs/superpowers/specs/2026-05-27-unified-audiobook-collections-design.md` §4.4.
**Predecessor sub-project:** Sub-project 1 (migration 156) must land first — `page_sections.media_types` is required.
---
## File map
**Modify:**
- `internal/sections/types.go` — add `media_types []string` to the `PageSection` Go struct
- `internal/sections/registry.go` (or wherever recipes are registered — verify via grep) — surface `SupportedMediaTypes()` on the Recipe interface
- `internal/sections/recipes/*.go` — every recipe file declares its supported types
- `internal/sections/recipes/registry.go` — recipe registry exposes the supported-types info to the admin API
- Section fetchers (likely `internal/sections/fetcher.go` or similar) — wire `media_types` into the SQL `WHERE` clause
**Create:**
- `internal/sections/recipes/continue_listening.go` + test
- `internal/sections/recipes/by_audiobook_series.go` + test
**Add to admin API:**
- `internal/api/handlers/recipes.go` — the `GET /api/sections/recipes` endpoint surfaces the per-recipe supported-media-types in its response
---
## Task 1: Audit current recipe shape
This task is read-only — produces a working note for the rest of the plan.
- [ ] **Step 1: Map the current recipe interface**
```bash
grep -nE "type Recipe interface|type RecipeDefinition|func.*Type\(\) string|func.*Definition\(\)|SupportedMediaTypes" internal/sections/recipes/*.go internal/sections/types.go internal/sections/registry.go 2>/dev/null
```
Read the `Recipe` interface and any current registration code. Note:
- The exact method signatures recipes already implement.
- How a recipe is registered (`init()` calls? explicit `Register`? slice literal?).
- Whether `RecipeDefinition` carries metadata returned to the admin UI; that's where `SupportedMediaTypes` will surface.
- [ ] **Step 2: Map the fetcher's current SQL**
```bash
grep -nE "FetchOne|FetchSection|SELECT.*FROM media_items|type Fetcher" internal/sections/*.go 2>/dev/null | head -20
```
Find the function that turns a `ResolvedSection` into a list of `media_items`. Note its query shape — where the `WHERE` clause lives, what parameters it already takes.
- [ ] **Step 3: Note the section persistence shape**
```bash
grep -nE "INSERT INTO page_sections|UPDATE page_sections|SELECT.*FROM page_sections" internal/sections/*.go internal/api/handlers/sections*.go 2>/dev/null | head -10
```
Find where `page_sections` rows are read and written. The `media_types` column needs to be read/written there too.
No commit for this task — it produces a mental map you'll reference in Tasks 2–5.
---
## Task 2: Add `media_types` to the `PageSection` struct + serde
**Files:**
- Modify: `internal/sections/types.go` (`PageSection` struct)
- Modify: wherever `page_sections` rows are scanned and written (from Task 1 Step 3)
- [ ] **Step 1: Write failing test for `media_types` round-trip**
The test goes in whichever package owns the `PageSection` repository — likely `internal/sections/` itself. If a repository test file exists, append; otherwise create one:
```go
func TestPageSectionMediaTypesRoundTrip(t *testing.T) {
if testing.Short() { t.Skip("requires test DB") }
ctx := context.Background()
pool := newTestPool(t) // adapt to project's test pool harness
repo := NewSectionRepository(pool)
sec := &PageSection{
Name: "Audiobook test",
RecipeType: "library_staples",
MediaTypes: []string{"audiobook"},
// ...fill in other required fields from the struct definition
}
if err := repo.Create(ctx, sec); err != nil {
t.Fatalf("Create: %v", err)
}
got, err := repo.GetByID(ctx, sec.ID)
if err != nil {
t.Fatalf("GetByID: %v", err)
}
if !reflect.DeepEqual(got.MediaTypes, []string{"audiobook"}) {
t.Errorf("MediaTypes = %v, want [audiobook]", got.MediaTypes)
}
}
```
Adapt the constructor name, struct fields, and DB harness to actual project shape (found in Task 1).
- [ ] **Step 2: Run the test to confirm failure**
Run: `go test ./internal/sections/ -run TestPageSectionMediaTypesRoundTrip -v`
Expected: FAIL — the `MediaTypes` field doesn't exist yet.
- [ ] **Step 3: Add the field to `PageSection`**
In `internal/sections/types.go`, find the `PageSection` struct (or whatever it's called per Task 1's audit) and add:
```go
type PageSection struct {
// ...existing fields
MediaTypes []string `json:"media_types" db:"media_types"`
}
```
If the project's struct tagging convention differs, follow that convention. The DB column is `text[]`; the Go type is `[]string`. pgx's default decoder handles this directly.
- [ ] **Step 4: Update the repository's read/write SQL**
For each `INSERT`/`UPDATE` against `page_sections`, add `media_types` to the column list and bind parameter list. For each `SELECT`, add it to the projection and scan into the new field.
Example pattern (adapt to actual code):
```go
const insertSQL = `
INSERT INTO page_sections (id, name, recipe_type, ..., media_types)
VALUES ($1, $2, $3, ..., $N::text[])
`
// ...
_, err := pool.Exec(ctx, insertSQL, sec.ID, sec.Name, sec.RecipeType, ..., sec.MediaTypes)
```
For SELECTs, add `media_types` to the projection and `&sec.MediaTypes` to the scan list.
- [ ] **Step 5: Run the test, verify pass**
Run: `go test ./internal/sections/ -run TestPageSectionMediaTypesRoundTrip -v`
Expected: PASS.
- [ ] **Step 6: Build + broader test**
```bash
go build ./...
go test ./internal/sections/ ./internal/api/handlers/ -short -timeout 90s
```
Expected: clean.
- [ ] **Step 7: Commit**
```bash
git add internal/sections/types.go internal/sections/*.go internal/api/handlers/sections*.go
git commit -m "feat(sections): expose page_sections.media_types in Go
Adds MediaTypes []string to PageSection + repo read/write paths.
Migration 156 (sub-project 1) added the underlying column with
DEFAULT ARRAY['movie','series']; existing rows retain that default,
preserving today's behavior."
```
Only stage the files you actually modified. Verify with `git diff --cached --stat` before commit.
---
## Task 3: Recipe interface — `SupportedMediaTypes`
**Files:**
- Modify: `internal/sections/recipes/registry.go` (interface declaration)
- Modify: every `*.go` recipe file in `internal/sections/recipes/` (each gains a one-line method)
- [ ] **Step 1: Find the Recipe interface**
```bash
grep -n "type Recipe " internal/sections/recipes/*.go internal/sections/*.go
```
Note the file + line of the interface declaration.
- [ ] **Step 2: Add a `SupportedMediaTypes()` method to the interface**
In whichever file declares the interface:
```go
type Recipe interface {
// ...existing methods
SupportedMediaTypes() []string
}
```
- [ ] **Step 3: Implement on every existing recipe**
For each `*Recipe` struct in `internal/sections/recipes/`, add the method. Existing recipes default to movies+series to preserve behavior:
```go
func (libStaple) SupportedMediaTypes() []string { return []string{"movie", "series"} }
func (moodRecipe) SupportedMediaTypes() []string { return []string{"movie", "series"} }
func (handPickedRecipe) SupportedMediaTypes() []string { return []string{"movie", "series"} }
// ...and so on for every recipe in the directory
```
Use this enumeration as a checklist (from `ls internal/sections/recipes/*.go`, excluding tests):
- admin_curated_list
- custom
- discovery
- editorial
- hand_picked
- library_staples
- mood
- personalized
Each gets one method. Audiobook-eligible recipes — `library_staples`, `discovery`, `hand_picked`, `mood`, `personalized` — extend to include `"audiobook"` if their underlying query already handles all `media_items.type` values transparently. Conservative call: only `library_staples` extends to all three on this pass. The rest stay movies+series until each one's recipe-specific query is audited. **Bias toward the conservative default.**
Specifically:
```go
func (libStaple) SupportedMediaTypes() []string { return []string{"movie", "series", "audiobook"} }
```
- [ ] **Step 4: Build**
```bash
go build ./...
```
Expected: clean. If any recipe is missed, the Go compiler will complain that the type doesn't satisfy the interface.
- [ ] **Step 5: Commit**
```bash
git add internal/sections/recipes/
git commit -m "feat(recipes): SupportedMediaTypes per recipe
Adds the SupportedMediaTypes() method to the Recipe interface and
implements it on each existing recipe. Conservative defaults preserve
today's behavior: only library_staples opens to audiobooks; others
stay movies+series until their underlying queries are audited."
```
---
## Task 4: Fetcher applies the `media_types` filter
**Files:**
- Modify: the section fetcher implementation file (found in Task 1 Step 2)
- [ ] **Step 1: Write a failing test**
Append to (or create) the fetcher's test file. Pattern:
```go
func TestFetcherFiltersByMediaTypes(t *testing.T) {
if testing.Short() { t.Skip("requires test DB") }
ctx := context.Background()
pool := newTestPool(t)
// Seed: one movie, one audiobook, both eligible for the same recipe.
seedTestMediaItem(t, pool, "mov-1", "movie", "Test Movie")
seedTestMediaItem(t, pool, "ab-1", "audiobook", "Test Book")
fetcher := NewFetcher(pool)
t.Run("media_types movies only", func(t *testing.T) {
got, _ := fetcher.FetchOne(ctx, ResolvedSection{
SectionType: "library_staples",
MediaTypes: []string{"movie"},
}, nil, nil, 1, "", catalog.AccessFilter{})
assertContains(t, got.Items, "mov-1")
assertExcludes(t, got.Items, "ab-1")
})
t.Run("media_types audiobook only", func(t *testing.T) {
got, _ := fetcher.FetchOne(ctx, ResolvedSection{
SectionType: "library_staples",
MediaTypes: []string{"audiobook"},
}, nil, nil, 1, "", catalog.AccessFilter{})
assertContains(t, got.Items, "ab-1")
assertExcludes(t, got.Items, "mov-1")
})
t.Run("media_types both", func(t *testing.T) {
got, _ := fetcher.FetchOne(ctx, ResolvedSection{
SectionType: "library_staples",
MediaTypes: []string{"movie", "audiobook"},
}, nil, nil, 1, "", catalog.AccessFilter{})
assertContains(t, got.Items, "mov-1", "ab-1")
})
}
```
Adapt `ResolvedSection`, `FetchOne` signature, `seedTestMediaItem`, `assertContains` / `assertExcludes` to the project's actual shapes. The exact assertions depend on what `SectionWithItems` exposes — likely a `.Items []SectionItem` slice.
- [ ] **Step 2: Run the test, verify fail**
```bash
go test ./internal/sections/ -run TestFetcherFiltersByMediaTypes -v
```
Expected: FAIL — `ResolvedSection.MediaTypes` field doesn't exist yet OR the fetcher SQL doesn't filter.
- [ ] **Step 3: Add `MediaTypes` to `ResolvedSection`**
In `internal/sections/types.go` (or wherever `ResolvedSection` lives):
```go
type ResolvedSection struct {
// ...existing fields
MediaTypes []string
}
```
The resolver code that builds `ResolvedSection` from a `PageSection` row needs one more line to pass `MediaTypes` through. Find it:
```bash
grep -n "ResolvedSection{" internal/sections/*.go internal/api/handlers/*.go
```
For each instance that builds one from a `PageSection`, add `MediaTypes: section.MediaTypes,`.
- [ ] **Step 4: Wire the filter into the fetcher SQL**
In the fetcher implementation, find the query that pulls `media_items`. Add to its `WHERE` clause:
```go
const query = `
SELECT ...
FROM media_items mi
WHERE ...
AND mi.type = ANY($N::text[])
...
`
// ...
rows, err := pool.Query(ctx, query, ..., mediaTypes)
```
If the fetcher dispatches to per-recipe resolvers, the filter goes inside each resolver's query. Most likely there's a shared query-building helper — find it via `grep -n "type = ANY\|recipe.*Resolve" internal/sections/`.
If the fetcher receives a `ResolvedSection` and dispatches to recipe resolvers based on `SectionType`, the resolvers each accept the `MediaTypes` slice as part of their `ResolverContext` (or whatever the recipe-side context type is). Add a field to that context type and thread it through.
**Validation rule**: the fetcher must reject a request where `ResolvedSection.MediaTypes` contains a type not in the recipe's `SupportedMediaTypes()`. Add this guard in the dispatch path:
```go
recipe, ok := registry.Get(resolved.SectionType)
if !ok { return result, fmt.Errorf("unknown recipe %q", resolved.SectionType) }
supported := setOf(recipe.SupportedMediaTypes())
for _, mt := range resolved.MediaTypes {
if _, ok := supported[mt]; !ok {
return result, fmt.Errorf("recipe %q does not support media type %q", resolved.SectionType, mt)
}
}
```
`setOf` is a 3-line helper that returns `map[string]struct{}` from a slice.
- [ ] **Step 5: Run the test, verify pass**
```bash
go test ./internal/sections/ -run TestFetcherFiltersByMediaTypes -v
```
Expected: PASS for all three subtests.
- [ ] **Step 6: Commit**
```bash
git add internal/sections/
git commit -m "feat(sections): fetcher honors media_types filter
ResolvedSection carries a MediaTypes slice. Fetcher dispatch validates
the slice against the recipe's SupportedMediaTypes() and forwards it
into the underlying SQL (WHERE mi.type = ANY(\$N::text[]))."
```
---
## Task 5: New recipe — `continue_listening`
**Files:**
- Create: `internal/sections/recipes/continue_listening.go`
- Create: `internal/sections/recipes/continue_listening_test.go`
The audiobook analog of `continue_watching`. Surfaces audiobook items the user has progress on but hasn't finished.
- [ ] **Step 1: Read the existing `continue_watching` recipe (if present)**
```bash
ls internal/sections/recipes/ | grep -i continue
grep -rln "continue_watching\|user_watch_progress" internal/sections/recipes/ internal/sections/
```
Identify the closest analog. Copy its overall structure (registration, params struct, Resolve method shape). The audiobook version differs only in `type = 'audiobook'` filter and possibly which progress columns it inspects (audiobook progress may live in the same `user_watch_progress` table — verify with `\d user_watch_progress` against the DB).
- [ ] **Step 2: Write a failing test**
```go
package recipes
import (
"context"
"testing"
)
func TestContinueListeningRecipeRegistered(t *testing.T) {
r, ok := registry.Get("continue_listening")
if !ok {
t.Fatal("continue_listening recipe not registered")
}
got := r.SupportedMediaTypes()
want := []string{"audiobook"}
if !equalStringSlices(got, want) {
t.Errorf("SupportedMediaTypes = %v, want %v", got, want)
}
}
func TestContinueListeningResolvesProgressedAudiobooks(t *testing.T) {
if testing.Short() { t.Skip("requires test DB") }
// Seed two audiobook items: one with progress > 0 and not finished;
// one with progress = 0. Run the recipe's Resolve method. Assert only
// the first is returned.
t.Skip("Implement once existing continue_watching test pattern is read")
}
```
- [ ] **Step 3: Implement the recipe**
```go
// Package recipes
package recipes
import (
"context"
"encoding/json"
"time"
)
type continueListeningRecipe struct{}
type ContinueListeningParams struct {
// e.g. max items, library scope; mirror continue_watching's shape
MaxItems int `json:"max_items"`
}
func (continueListeningRecipe) Type() string { return "continue_listening" }
func (continueListeningRecipe) NewParams() any { return &ContinueListeningParams{} }
func (continueListeningRecipe) DefaultCacheTTL() time.Duration { return 5 * time.Minute }
func (continueListeningRecipe) SupportedMediaTypes() []string { return []string{"audiobook"} }
func (continueListeningRecipe) Resolve(rc ResolverContext) (ResolvedItems, error) {
const q = `
SELECT mi.content_id, mi.title, mi.year, mi.poster_path
FROM media_items mi
JOIN user_watch_progress uwp ON uwp.content_id = mi.content_id
WHERE mi.type = 'audiobook'
AND uwp.user_id = $1
AND uwp.position_seconds > 0
AND COALESCE(uwp.completed, false) = false
ORDER BY uwp.updated_at DESC
LIMIT $2
`
params := rc.Params.(*ContinueListeningParams)
limit := params.MaxItems
if limit <= 0 || limit > 50 { limit = 20 }
rows, err := rc.Pool.Query(rc.Ctx, q, rc.UserID, limit)
if err != nil { return ResolvedItems{}, fmt.Errorf("continue_listening: %w", err) }
defer rows.Close()
var items []ResolvedItem
for rows.Next() {
var it ResolvedItem
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &it.PosterPath); err != nil {
return ResolvedItems{}, fmt.Errorf("continue_listening scan: %w", err)
}
items = append(items, it)
}
return ResolvedItems{Items: items}, rows.Err()
}
func (continueListeningRecipe) Validate(raw json.RawMessage) error {
var p ContinueListeningParams
return json.Unmarshal(raw, &p)
}
func (continueListeningRecipe) Definition() RecipeDefinition {
return RecipeDefinition{
Type: "continue_listening",
Name: "Continue Listening",
Description: "Audiobooks you've started but haven't finished.",
}
}
func init() {
registry.Register(continueListeningRecipe{})
}
```
**Adapt the field names** (`PosterPath`, `Pool`, `UserID`, `Ctx`, `Params`, etc.) to whatever the existing recipes use. **Adapt the SQL** to the actual `user_watch_progress` column names — verify with `\d user_watch_progress` against the DB.
- [ ] **Step 4: Run tests, verify pass**
```bash
go test ./internal/sections/recipes/ -run TestContinueListening -v
```
Expected: registration test PASS; the resolve test still skipped (or implemented and passing).
- [ ] **Step 5: Commit**
```bash
git add internal/sections/recipes/continue_listening.go internal/sections/recipes/continue_listening_test.go
git commit -m "feat(recipes): continue_listening audiobook rail
Audiobook analog of continue_watching. Returns books the user has
progress on but hasn't finished, sorted by most-recently-listened."
```
---
## Task 6: New recipe — `by_audiobook_series`
**Files:**
- Create: `internal/sections/recipes/by_audiobook_series.go`
- Create: `internal/sections/recipes/by_audiobook_series_test.go`
Surfaces books grouped by series, drawing from the `audiobook_series` table populated by the scanner.
- [ ] **Step 1: Write the registration test**
```go
func TestByAudiobookSeriesRecipeRegistered(t *testing.T) {
r, ok := registry.Get("by_audiobook_series")
if !ok { t.Fatal("by_audiobook_series not registered") }
if got := r.SupportedMediaTypes(); !equalStringSlices(got, []string{"audiobook"}) {
t.Errorf("SupportedMediaTypes = %v, want [audiobook]", got)
}
}
```
- [ ] **Step 2: Implement the recipe**
```go
type byAudiobookSeriesRecipe struct{}
type ByAudiobookSeriesParams struct {
SeriesName string `json:"series_name"`
}
func (byAudiobookSeriesRecipe) Type() string { return "by_audiobook_series" }
func (byAudiobookSeriesRecipe) NewParams() any { return &ByAudiobookSeriesParams{} }
func (byAudiobookSeriesRecipe) DefaultCacheTTL() time.Duration { return 30 * time.Minute }
func (byAudiobookSeriesRecipe) SupportedMediaTypes() []string { return []string{"audiobook"} }
func (byAudiobookSeriesRecipe) Resolve(rc ResolverContext) (ResolvedItems, error) {
const q = `
SELECT mi.content_id, mi.title, mi.year, mi.poster_path
FROM media_items mi
JOIN audiobook_series s ON s.content_id = mi.content_id
WHERE mi.type = 'audiobook'
AND s.series_name = $1
ORDER BY COALESCE(s.series_index, 9999), mi.title
`
params := rc.Params.(*ByAudiobookSeriesParams)
if params.SeriesName == "" {
return ResolvedItems{}, fmt.Errorf("by_audiobook_series: series_name required")
}
rows, err := rc.Pool.Query(rc.Ctx, q, params.SeriesName)
if err != nil { return ResolvedItems{}, fmt.Errorf("by_audiobook_series: %w", err) }
defer rows.Close()
var items []ResolvedItem
for rows.Next() {
var it ResolvedItem
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &it.PosterPath); err != nil {
return ResolvedItems{}, fmt.Errorf("by_audiobook_series scan: %w", err)
}
items = append(items, it)
}
return ResolvedItems{Items: items}, rows.Err()
}
func (byAudiobookSeriesRecipe) Validate(raw json.RawMessage) error {
var p ByAudiobookSeriesParams
if err := json.Unmarshal(raw, &p); err != nil { return err }
if p.SeriesName == "" { return fmt.Errorf("series_name is required") }
return nil
}
func (byAudiobookSeriesRecipe) Definition() RecipeDefinition {
return RecipeDefinition{
Type: "by_audiobook_series",
Name: "By Audiobook Series",
Description: "Books in a specific audiobook series, ordered by series position.",
}
}
func init() {
registry.Register(byAudiobookSeriesRecipe{})
}
```
- [ ] **Step 3: Run test, verify pass**
```bash
go test ./internal/sections/recipes/ -run TestByAudiobookSeries -v
```
Expected: PASS.
- [ ] **Step 4: Commit**
```bash
git add internal/sections/recipes/by_audiobook_series.go internal/sections/recipes/by_audiobook_series_test.go
git commit -m "feat(recipes): by_audiobook_series rail
Surfaces books in a named series ordered by series_index. Backed by
the audiobook_series table populated by the scanner."
```
---
## Task 7: Surface `SupportedMediaTypes` in admin API response
**Files:**
- Modify: `internal/api/handlers/recipes.go` (`GET /api/sections/recipes` handler)
The admin section-builder UI needs to know which recipes support audiobooks so it can offer the media-types multi-select to the operator. The handler's response shape adds one field per recipe.
- [ ] **Step 1: Read the current handler**
```bash
cat internal/api/handlers/recipes.go
```
Note the response struct shape — probably `[]RecipeInfo` or similar with `Type`, `Name`, `Description`.
- [ ] **Step 2: Add `SupportedMediaTypes` to the response**
In the response struct definition, add:
```go
type recipeInfo struct {
Type string `json:"type"`
Name string `json:"name"`
Description string `json:"description"`
SupportedMediaTypes []string `json:"supported_media_types"`
}
```
In the handler body, set `SupportedMediaTypes: recipe.SupportedMediaTypes()`.
- [ ] **Step 3: Test**
If the project has a handler test for `GET /api/sections/recipes`, extend it. Otherwise the manual verification step covers this:
```bash
curl -sH "Authorization: Bearer $TOKEN" http://localhost:8090/api/sections/recipes | jq '.[] | select(.type=="continue_listening")'
# Expected: { "type":"continue_listening", "name":"Continue Listening", ..., "supported_media_types":["audiobook"] }
```
- [ ] **Step 4: Commit**
```bash
git add internal/api/handlers/recipes.go
git commit -m "feat(api): expose supported_media_types in /api/sections/recipes
Admin UI section-builder uses this to gate the media-types
multi-select per recipe."
```
---
## Verification (after merge)
1. Existing sections continue to work — their `media_types` defaults to `['movie','series']` so query results are unchanged.
2. Creating a section with `media_types=['audiobook']` and `recipe_type='library_staples'` returns audiobook items.
3. Two new recipes registered:
```bash
curl -sH "Authorization: Bearer $TOKEN" http://localhost:8090/api/sections/recipes | jq '[.[] | .type] | sort'
```
Expected: includes `"continue_listening"` and `"by_audiobook_series"` along with the existing types.
4. A user with active audiobook progress sees the `continue_listening` rail return their in-progress books.
5. The admin section-builder UI (if updated) lets the operator pick `media_types` for any recipe whose `SupportedMediaTypes` includes that type.
---
## Self-Review
**Spec coverage:**
- `page_sections.media_types` Go-side serde ✓ (Task 2)
- `Recipe.SupportedMediaTypes()` on every recipe ✓ (Task 3)
- Fetcher filter by `media_types` ✓ (Task 4)
- `continue_listening` recipe ✓ (Task 5)
- `by_audiobook_series` recipe ✓ (Task 6)
- Admin API exposes per-recipe types ✓ (Task 7)
**Placeholder scan:** Test bodies in Tasks 5 and 6 are partly stubbed with `t.Skip(...)` to be filled in by reading the existing `continue_watching` test pattern. That's a deliberate "read-existing-then-write" instruction, not a TBD. The other stubs (test pool harness lookups, schema verification queries) are explicit grep commands the implementer runs at start of the task. No abstract "implement appropriately."
**Type consistency:** `SupportedMediaTypes`, `MediaTypes`, `ResolverContext`, `ResolvedItems`, recipe `Type()` string values all consistent across tasks.
**Risk:** Lowest-risk of the 4 sub-projects. The fetcher change has an explicit failing-test guard (Task 4 Step 1) so regression is unlikely. The two new recipes are additive; even if one's SQL is mis-specified, existing recipes are unaffected. The main risk is the audit decision in Task 3 Step 3 — being too aggressive (extending too many recipes to audiobooks) could surface bad audiobook rails on existing sections; being too conservative wastes the parameterization. Conservative is the safer default.
docs/superpowers/plans/2026-06-01-request-multi-instance-arr-routing.md
+1704
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-06-02-autoscan-arr-polling.md
+1198
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-06-02-autoscan-host-backend.md
+779
View File
@@ -0,0 +1,779 @@
# Autoscan Host Backend Implementation Plan (Part 1 of 2)
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Build the silo-server backend for the standalone Autoscan category: poll `scan_source.v1` plugins on a timer, resolve their Silo-native paths to library folders, and enqueue rescans — with connections decoupled from Requests.
**Architecture:** Mirror the existing `scheduled_task.v1` host dispatch (`pluginhost.Client` capability wrapper + `plugins.Service` resolver + a `taskmanager` task). Generalize the closed PR #43 autoscan engine (resolve → suppress → enqueue) so its input is "Silo-native paths from a scan-source plugin" instead of "arr history." Replace the old `request_integrations`-coupled schema with `autoscan_connections` + `autoscan_sources`.
**Tech Stack:** Go, pgx/v5, the `internal/plugins`/`internal/pluginhost` runtime, `internal/scantrigger`/`internal/scanqueue`, `internal/taskmanager`, numbered SQL migrations, `github.com/Silo-Server/silo-plugin-sdk` (the new `scan_source.v1` capability).
Commands assume the silo-server repository root is the cwd. This is **Part 1 (backend)**; the Autoscan admin UI is Part 2.
---
## Prerequisites
1. **SDK dependency.** This plan needs the SDK's `scan_source.v1` symbols (`runtime.Client.ScanSource()`, `pluginv1.ScanSourceClient`, `pluginv1.PollChangesRequest/Response`, `capability.ScanSource`). They are in `silo-plugin-sdk` PR #2 but **not yet released**. Until `v0.5.0` is tagged, add a local replace directive so the host builds against the working checkout:
```bash
# from the repository root (cwd)
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk
go mod tidy
```
This points the replace at the `silo-plugin-sdk` checkout (a sibling of this repo).
The final task removes the replace and bumps to the tagged version.
2. **Closed PR #43 is on `main`.** The in-process arr autoscan (`internal/autoscan/*`, `internal/api/handlers/autoscan.go`, `internal/taskmanager/tasks/autoscan_poll.go`, router routes, migration `171_autoscan`, the `AdminRequests.tsx` tab) is the salvage source. This plan **generalizes the engine and replaces the schema/wiring**; it does not preserve the arr-specific history client (that moves to the arr plugin in a separate plan).
3. **Tests that touch `internal/api/handlers` or `internal/api` link libvips via CGO.** Run the suite in the libvips-equipped container (see project memory `silo-test-libvips-gotcha`), not the bare host, or those packages report `[build failed]` and skip silently.
---
## Pattern references (read before starting)
- Host capability client wrapper: `internal/pluginhost/client.go` — `ScheduledTaskClient` type (l.42), `Client.ScheduledTask` accessor (l.111), `requireCapability` (l.157), `Run` (l.226).
- Host resolver: `internal/plugins/service.go:413` `ScheduledTaskClient(ctx, installationID, capabilityID)` → `ensureClient` → `client.ScheduledTask(capabilityID)`.
- Timer task over plugin capabilities: `internal/plugins/task_registry.go` (iterate enabled installations, filter capability type, build a `taskmanager.Task`).
- Salvageable engine (generalize, don't rewrite): `internal/autoscan/service.go` (resolve/suppress/enqueue loop), `dedupe.go` (`uniqueParentDirs`), `suppress.go` (`Suppressor`). Discard `history.go`, `rewrite.go`, `suggest*.go` (arr-specific → arr plugin).
- Migration mechanics: paired `migrations/NNN_*.up.sql`/`.down.sql`, embedded via `migrations/embed.go`, auto-applied on boot.
- Credential decryption for the reuse-from-Requests case: `internal/requests/repository.go` (`integrationColumns` has `base_url, api_key_ref`); Fernet resolution via `catalog.ServerSettingsRepo.Get` (the same path PR #43's `SecretResolver` used).
---
## File Structure
| File | Responsibility | Action |
|---|---|---|
| `internal/pluginhost/client.go` | `ScanSourceClient` wrapper + `Client.ScanSource()` accessor | Modify |
| `internal/pluginhost/client_test.go` | `requireCapability` gate for `scan_source.v1` | Modify |
| `internal/plugins/service.go` | `ScanSourceClient(ctx, installationID, capabilityID)` resolver | Modify |
| `migrations/172_autoscan_v2.up.sql` / `.down.sql` | `autoscan_connections` + `autoscan_sources`; drop the `171` tables | Create |
| `internal/autoscan/types.go` | `Source`, `Connection`, `Settings`, `PollChanges` provider seam | Rewrite |
| `internal/autoscan/repository.go` | CRUD for connections + sources; `AdvanceMarker` | Rewrite |
| `internal/autoscan/provider.go` | `ScanSourceProvider` interface + adapter over the plugins resolver | Create |
| `internal/autoscan/connection.go` | Resolve a `Connection` (own or Requests-linked) → `{base_url, api_key}` | Create |
| `internal/autoscan/service.go` | Generic engine: per source → `PollChanges(marker)` → resolve → suppress → enqueue → store marker | Rewrite |
| `internal/autoscan/dedupe.go`, `suppress.go` | Unchanged salvage | Keep |
| `internal/autoscan/history.go`, `rewrite.go`, `suggest*.go` | Arr-specific | Delete |
| `internal/taskmanager/tasks/autoscan_poll.go` | Poll task → `Service.PollOnce` | Keep (retarget) |
| `internal/api/handlers/autoscan.go` | Admin API (settings/connections/sources/trigger/status) | Rewrite |
| `internal/api/router.go`, `cmd/silo/main.go` | Wire the rebuilt service + connection resolver; drop request-coupled wiring | Modify |
| `go.mod` | SDK replace (dev) → `v0.5.0` (final) | Modify |
---
## Task 1: Host `pluginhost` client wrapper for `scan_source.v1`
**Files:**
- Modify: `internal/pluginhost/client.go`
- Modify: `internal/pluginhost/client_test.go`
- [ ] **Step 1: Write the failing test**
In `internal/pluginhost/client_test.go`, add a test mirroring the existing `scheduled_task` capability-gate test (find it first; it constructs a `Client` with a manifest declaring a capability and asserts the accessor errors when the capability is absent and succeeds when present):
```go
func TestClientScanSourceRequiresCapability(t *testing.T) {
// Client whose manifest does NOT declare scan_source.v1
c := newTestClientWithCapabilities(t, "metadata_provider.v1")
if _, err := c.ScanSource("missing"); err == nil {
t.Fatal("ScanSource: expected error when capability absent")
}
// Client WHOSE manifest declares scan_source.v1 with id "arr"
c2 := newTestClientWithCapabilities(t, "scan_source.v1:arr")
if _, err := c2.ScanSource("arr"); err != nil {
t.Fatalf("ScanSource: unexpected error when capability present: %v", err)
}
}
```
Use whatever helper the sibling `scheduled_task` test uses to construct a `Client` with declared capabilities; mirror it exactly (do not invent a helper that does not exist — read `client_test.go` and copy the construction).
- [ ] **Step 2: Run test to verify it fails**
Run: `go test ./internal/pluginhost/ -run TestClientScanSourceRequiresCapability`
Expected: FAIL — `c.ScanSource undefined`.
- [ ] **Step 3: Add the wrapper type and accessor**
In `internal/pluginhost/client.go`, after the `ScheduledTaskClient` type (l.42), add:
```go
type ScanSourceClient struct {
client pluginv1.ScanSourceClient
timeout time.Duration
}
```
After the `Client.ScheduledTask` accessor (l.117), add:
```go
func (c *Client) ScanSource(capabilityID string) (*ScanSourceClient, error) {
if err := c.requireCapability("scan_source.v1", capabilityID); err != nil {
return nil, err
}
return &ScanSourceClient{
client: c.rpc.ScanSource(),
timeout: DefaultControlTimeout,
}, nil
}
func (c *ScanSourceClient) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
callCtx, cancel := ensureDeadline(ctx, c.timeout)
defer cancel()
return c.client.PollChanges(callCtx, req)
}
```
- [ ] **Step 4: Run test to verify it passes**
Run: `go test ./internal/pluginhost/ -run TestClientScanSourceRequiresCapability`
Expected: PASS.
- [ ] **Step 5: Commit**
```bash
git add internal/pluginhost/client.go internal/pluginhost/client_test.go
git commit -m "feat(pluginhost): scan_source.v1 capability client wrapper"
```
---
## Task 2: `plugins.Service` resolver for `scan_source.v1`
**Files:**
- Modify: `internal/plugins/service.go`
- [ ] **Step 1: Add the resolver method**
After `ScheduledTaskClient` (l.413–423), add the identical shape for scan source:
```go
func (s *Service) ScanSourceClient(
ctx context.Context,
installationID int,
capabilityID string,
) (*pluginhost.ScanSourceClient, error) {
client, err := s.ensureClient(ctx, installationID)
if err != nil {
return nil, err
}
return client.ScanSource(capabilityID)
}
```
- [ ] **Step 2: Build to verify it compiles**
Run: `go build ./internal/plugins/`
Expected: exits 0.
- [ ] **Step 3: Commit**
```bash
git add internal/plugins/service.go
git commit -m "feat(plugins): expose scan_source.v1 client resolver"
```
---
## Task 3: Schema — `autoscan_connections` + `autoscan_sources`
**Files:**
- Create: `migrations/172_autoscan_v2.up.sql`, `migrations/172_autoscan_v2.down.sql`
- [ ] **Step 1: Write the up migration**
Create `migrations/172_autoscan_v2.up.sql`. It drops PR #43's `171` tables and creates the decoupled schema (no FK into `request_integrations`; the optional reuse link is `ON DELETE SET NULL`):
```sql
DROP TABLE IF EXISTS public.autoscan_sources;
DROP TABLE IF EXISTS public.autoscan_settings;
CREATE TABLE public.autoscan_settings (
id boolean PRIMARY KEY DEFAULT true,
enabled boolean NOT NULL DEFAULT false,
default_poll_interval_seconds integer NOT NULL DEFAULT 600,
debounce_seconds integer NOT NULL DEFAULT 60,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT autoscan_settings_singleton CHECK (id),
CONSTRAINT autoscan_settings_interval_pos CHECK (default_poll_interval_seconds > 0),
CONSTRAINT autoscan_settings_debounce_nonneg CHECK (debounce_seconds >= 0)
);
INSERT INTO public.autoscan_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
CREATE TABLE public.autoscan_connections (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
name text NOT NULL,
kind text NOT NULL,
-- Own credentials (used when request_integration_id IS NULL):
base_url text,
api_key_ref text,
-- Optional soft reuse of a Requests arr server (live link; SET NULL on delete):
request_integration_id text REFERENCES public.request_integrations(id) ON DELETE SET NULL,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT autoscan_connections_source_present
CHECK (request_integration_id IS NOT NULL OR base_url IS NOT NULL)
);
CREATE TABLE public.autoscan_sources (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
installation_id integer NOT NULL,
capability_id text NOT NULL,
connection_id uuid NOT NULL REFERENCES public.autoscan_connections(id) ON DELETE RESTRICT,
enabled boolean NOT NULL DEFAULT false,
poll_interval_seconds integer,
marker text,
last_run_at timestamptz,
last_error text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT autoscan_sources_interval_pos CHECK (poll_interval_seconds IS NULL OR poll_interval_seconds > 0),
CONSTRAINT autoscan_sources_capability_unique UNIQUE (installation_id, capability_id)
);
```
- [ ] **Step 2: Write the down migration**
Create `migrations/172_autoscan_v2.down.sql`:
```sql
DROP TABLE IF EXISTS public.autoscan_sources;
DROP TABLE IF EXISTS public.autoscan_connections;
DROP TABLE IF EXISTS public.autoscan_settings;
```
(Recreating the `171` tables on down-migration is unnecessary: `171` is unreleased and superseded; the down simply removes `172`'s tables.)
- [ ] **Step 3: Apply and verify against a disposable DB**
Run the migrations against a throwaway Postgres (the project's disposable-DB pattern) and confirm boot applies cleanly:
```bash
docker compose up -d postgres
go run ./cmd/silo --migrate-only 2>&1 | grep -i "migrations applied" || echo "check migration output"
```
Expected: migrations apply without error; `\d autoscan_connections` shows the soft FK `request_integration_id`.
- [ ] **Step 4: Commit**
```bash
git add migrations/172_autoscan_v2.up.sql migrations/172_autoscan_v2.down.sql
git commit -m "feat(migrations): autoscan v2 schema (connections + sources, no requests FK)"
```
---
## Task 4: Types and repository
**Files:**
- Rewrite: `internal/autoscan/types.go`
- Rewrite: `internal/autoscan/repository.go`
- [ ] **Step 1: Define the types**
Replace `internal/autoscan/types.go` with the v2 model:
```go
package autoscan
import "time"
type Settings struct {
Enabled bool
DefaultPollIntervalSeconds int
DebounceSeconds int
}
// Connection is an arr server the host can reach: either own credentials, or a
// live reference to a Requests integration (RequestIntegrationID set).
type Connection struct {
ID string
Name string
Kind string
BaseURL string // own; empty when linked
APIKeyRef string // own; empty when linked
RequestIntegrationID *string
}
// Source ties a scan_source plugin capability instance to a connection plus the
// host-owned scheduling/bookkeeping state.
type Source struct {
ID string
InstallationID int
CapabilityID string
ConnectionID string
Enabled bool
PollIntervalSeconds *int // nil => use settings default
Marker *string // opaque; nil on first run
LastRunAt *time.Time
LastError *string
}
```
- [ ] **Step 2: Write a failing repository test**
Add `internal/autoscan/repository_test.go` exercising round-trip + `AdvanceMarker` against a disposable DB (gate with the project's DB-test build tag/helper if one exists; otherwise a `//go:build integration` tag matching sibling DB tests). Minimal:
```go
func TestRepositoryMarkerRoundTrip(t *testing.T) {
repo := newTestRepo(t) // disposable-DB helper used by sibling repo tests
connID := repo.mustCreateConnection(t, Connection{Name: "Sonarr", Kind: "sonarr", BaseURL: "http://x", APIKeyRef: "k"})
srcID := repo.mustCreateSource(t, Source{InstallationID: 1, CapabilityID: "arr", ConnectionID: connID, Enabled: true})
if err := repo.AdvanceMarker(ctx, srcID, "2026-06-02T14:10:00Z"); err != nil {
t.Fatalf("AdvanceMarker: %v", err)
}
got, _ := repo.ListEnabledSources(ctx)
if len(got) != 1 || got[0].Marker == nil || *got[0].Marker != "2026-06-02T14:10:00Z" {
t.Fatalf("marker not persisted: %+v", got)
}
}
```
- [ ] **Step 3: Run to verify it fails**
Run: `go test ./internal/autoscan/ -run TestRepositoryMarkerRoundTrip` (with the DB helper available)
Expected: FAIL — undefined repository methods.
- [ ] **Step 4: Implement the repository**
Rewrite `internal/autoscan/repository.go` with: `GetSettings`, `UpdateSettings`, connection CRUD (`CreateConnection`, `UpdateConnection`, `DeleteConnection`, `ListConnections`, `GetConnection`), source CRUD (`UpsertSource`, `ListEnabledSources`, `ListSources`, `GetSource`), and `AdvanceMarker(ctx, sourceID, marker string)` (sets `marker`, `last_run_at = now()`, clears `last_error`). Use pgx; follow the column/scan style of the existing (pre-rewrite) `repository.go` and `internal/requests/repository.go`.
- [ ] **Step 5: Run to verify it passes**
Run: `go test ./internal/autoscan/ -run TestRepositoryMarkerRoundTrip`
Expected: PASS.
- [ ] **Step 6: Commit**
```bash
git add internal/autoscan/types.go internal/autoscan/repository.go internal/autoscan/repository_test.go
git commit -m "feat(autoscan): v2 types and repository (connections, sources, markers)"
```
---
## Task 5: Connection resolution (own or Requests-linked)
**Files:**
- Create: `internal/autoscan/connection.go`
- Create: `internal/autoscan/connection_test.go`
- [ ] **Step 1: Write the failing test**
```go
func TestResolveConnectionOwnVsLinked(t *testing.T) {
reqLookup := fakeRequestIntegrationLookup{"req-1": creds{BaseURL: "http://req:7878", APIKeyRef: "rk"}}
secrets := fakeSecrets{"rk": "REQKEY", "ownref": "OWNKEY"}
r := NewConnectionResolver(reqLookup, secrets)
own, err := r.Resolve(ctx, Connection{BaseURL: "http://own:8989", APIKeyRef: "ownref"})
if err != nil || own.BaseURL != "http://own:8989" || own.APIKey != "OWNKEY" {
t.Fatalf("own resolve = %+v, err=%v", own, err)
}
id := "req-1"
linked, err := r.Resolve(ctx, Connection{RequestIntegrationID: &id})
if err != nil || linked.BaseURL != "http://req:7878" || linked.APIKey != "REQKEY" {
t.Fatalf("linked resolve = %+v, err=%v", linked, err)
}
}
func TestResolveConnectionLinkedMissingFallsBack(t *testing.T) {
r := NewConnectionResolver(fakeRequestIntegrationLookup{}, fakeSecrets{})
id := "gone"
if _, err := r.Resolve(ctx, Connection{RequestIntegrationID: &id}); err == nil {
t.Fatal("expected error when linked Requests integration is missing")
}
}
```
- [ ] **Step 2: Run to verify it fails**
Run: `go test ./internal/autoscan/ -run TestResolveConnection`
Expected: FAIL — undefined `NewConnectionResolver`.
- [ ] **Step 3: Implement**
Create `internal/autoscan/connection.go`:
```go
package autoscan
import (
"context"
"fmt"
)
// ResolvedConnection is concrete credentials handed to the plugin.
type ResolvedConnection struct {
BaseURL string
APIKey string
}
type RequestIntegrationLookup interface {
// Returns base URL + encrypted api key ref for a Requests integration.
Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error)
}
type SecretResolver interface {
Get(ctx context.Context, ref string) (string, error)
}
type ConnectionResolver struct {
requests RequestIntegrationLookup
secrets SecretResolver
}
func NewConnectionResolver(r RequestIntegrationLookup, s SecretResolver) *ConnectionResolver {
return &ConnectionResolver{requests: r, secrets: s}
}
func (cr *ConnectionResolver) Resolve(ctx context.Context, c Connection) (ResolvedConnection, error) {
baseURL, apiKeyRef := c.BaseURL, c.APIKeyRef
if c.RequestIntegrationID != nil {
u, ref, err := cr.requests.Get(ctx, *c.RequestIntegrationID)
if err != nil {
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: %w", *c.RequestIntegrationID, err)
}
baseURL, apiKeyRef = u, ref
}
apiKey := apiKeyRef
if cr.secrets != nil && apiKeyRef != "" {
resolved, err := cr.secrets.Get(ctx, apiKeyRef)
if err != nil {
return ResolvedConnection{}, fmt.Errorf("autoscan: resolve api key: %w", err)
}
if resolved != "" {
apiKey = resolved
}
}
return ResolvedConnection{BaseURL: baseURL, APIKey: apiKey}, nil
}
```
- [ ] **Step 4: Run to verify it passes**
Run: `go test ./internal/autoscan/ -run TestResolveConnection`
Expected: PASS.
- [ ] **Step 5: Commit**
```bash
git add internal/autoscan/connection.go internal/autoscan/connection_test.go
git commit -m "feat(autoscan): resolve connections (own credentials or Requests-linked)"
```
---
## Task 6: Provider seam over the plugin resolver
**Files:**
- Create: `internal/autoscan/provider.go`
- [ ] **Step 1: Define the seam and adapter**
The engine must be testable without a live plugin, so it depends on a narrow interface, with a production adapter over `plugins.Service.ScanSourceClient`. Create `internal/autoscan/provider.go`:
```go
package autoscan
import (
"context"
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
)
// ScanSourceProvider yields changed paths for one source. The engine calls
// PollChanges; production wraps the plugins.Service scan_source resolver.
type ScanSourceProvider interface {
PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection) (paths []string, nextMarker string, err error)
}
// pluginScanSourceClient is the slice of *pluginhost.ScanSourceClient used here.
type pluginScanSourceClient interface {
PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error)
}
type scanSourceResolver interface {
ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (pluginScanSourceClient, error)
}
type pluginProvider struct{ resolver scanSourceResolver }
func NewPluginProvider(resolver scanSourceResolver) ScanSourceProvider {
return &pluginProvider{resolver: resolver}
}
func (p *pluginProvider) PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection) ([]string, string, error) {
client, err := p.resolver.ScanSourceClient(ctx, installationID, capabilityID)
if err != nil {
return nil, "", err
}
resp, err := client.PollChanges(ctx, &pluginv1.PollChangesRequest{CapabilityId: capabilityID, Marker: marker})
if err != nil {
return nil, "", err
}
return resp.GetChangedPaths(), resp.GetNextMarker(), nil
}
```
Note: the connection (`conn`) is delivered to the plugin out-of-band as runtime config when the source is configured (the plugin reads its own `{base_url, api_key}`); it is passed here so a future provider variant could inject per-call. For v1 the production path configures the plugin instance with the resolved connection at upsert time; document this in `NewPluginProvider`'s comment.
- [ ] **Step 2: Build**
Run: `go build ./internal/autoscan/`
Expected: exits 0.
- [ ] **Step 3: Commit**
```bash
git add internal/autoscan/provider.go
git commit -m "feat(autoscan): scan-source provider seam over the plugin resolver"
```
---
## Task 7: Generalize the engine (`Service.PollOnce`)
**Files:**
- Rewrite: `internal/autoscan/service.go`
- Modify: `internal/autoscan/service_test.go` (generalize the salvaged tests)
- [ ] **Step 1: Adapt the salvaged tests to the provider seam**
The PR #43 `service_test.go` used a `fakeHistory`; replace it with a `fakeProvider` implementing `ScanSourceProvider`, keeping the same assertions (dedupe to one folder; distinct paths same folder → two scans; disabled no-op; provider error keeps marker; enqueue failure releases claims + keeps marker). Add an opaque-marker assertion: the provider's returned `nextMarker` is stored verbatim via `AdvanceMarker`.
```go
type fakeProvider struct {
paths map[string][]string // key: capabilityID
nextMarker string
err error
}
func (f *fakeProvider) PollChanges(_ context.Context, _ int, capabilityID, _ string, _ ResolvedConnection) ([]string, string, error) {
if f.err != nil {
return nil, "", f.err
}
return f.paths[capabilityID], f.nextMarker, nil
}
```
- [ ] **Step 2: Run to verify the tests fail**
Run: `go test ./internal/autoscan/ -run TestPollOnce`
Expected: FAIL — `Service` signature/`fakeProvider` not yet wired.
- [ ] **Step 3: Rewrite the engine**
Rewrite `internal/autoscan/service.go`. Keep the salvaged resolve→suppress→enqueue loop verbatim (it is provider-agnostic already), but drive it from sources + the provider seam:
```go
func (s *Service) PollOnce(ctx context.Context) error {
settings, err := s.store.GetSettings(ctx)
if err != nil {
return err
}
if !settings.Enabled {
return nil
}
sources, err := s.store.ListEnabledSources(ctx)
if err != nil {
return err
}
ttl := time.Duration(settings.DebounceSeconds) * time.Second
for _, src := range sources {
conn, cerr := s.resolveConnection(ctx, src.ConnectionID)
if cerr != nil {
slog.WarnContext(ctx, "autoscan: resolve connection failed", "source_id", src.ID, "err", cerr)
continue
}
marker := ""
if src.Marker != nil {
marker = *src.Marker
}
paths, next, perr := s.provider.PollChanges(ctx, src.InstallationID, src.CapabilityID, marker, conn)
if perr != nil {
slog.WarnContext(ctx, "autoscan: poll changes failed", "source_id", src.ID, "err", perr)
_ = s.store.RecordError(ctx, src.ID, perr.Error())
continue
}
targets, claimed := s.resolveAndClaim(ctx, paths, ttl) // salvaged dedupe→resolve→suppress
if len(targets) > 0 {
if eerr := s.queue.EnqueueScans(ctx, targets); eerr != nil {
s.releaseClaims(ctx, claimed)
slog.WarnContext(ctx, "autoscan: enqueue failed", "source_id", src.ID, "err", eerr)
continue // do NOT advance marker
}
}
if aerr := s.store.AdvanceMarker(ctx, src.ID, next); aerr != nil {
slog.WarnContext(ctx, "autoscan: advance marker failed", "source_id", src.ID, "err", aerr)
}
}
return nil
}
```
Extract the salvaged loop body into `resolveAndClaim`/`releaseClaims` helpers (lifted verbatim from PR #43's `service.go` lines that build `targets`/`claimed`). Keep `uniqueParentDirs`, `Suppressor`, the `fmt.Sprintf("%d|%s", folderID, target.Path)` suppression key, and the `scantrigger.RequestError` quiet-skip.
- [ ] **Step 4: Run to verify the tests pass**
Run: `go test ./internal/autoscan/ -run TestPollOnce`
Expected: PASS.
- [ ] **Step 5: Commit**
```bash
git add internal/autoscan/service.go internal/autoscan/service_test.go
git commit -m "feat(autoscan): generic engine drives sources via scan_source provider"
```
---
## Task 8: Delete arr-specific salvage residue
**Files:**
- Delete: `internal/autoscan/history.go`, `history_test.go`, `rewrite.go`, `rewrite_test.go`, `suggest.go`, `suggest_deps.go`, `suggest_test.go`
- [ ] **Step 1: Remove the files**
```bash
git rm internal/autoscan/history.go internal/autoscan/history_test.go \
internal/autoscan/rewrite.go internal/autoscan/rewrite_test.go \
internal/autoscan/suggest.go internal/autoscan/suggest_deps.go internal/autoscan/suggest_test.go
```
- [ ] **Step 2: Build to confirm no remaining references**
Run: `go build ./internal/autoscan/...`
Expected: exits 0 (the engine no longer references the deleted arr logic). If it fails, a reference was missed — remove it.
- [ ] **Step 3: Commit**
```bash
git commit -m "refactor(autoscan): drop arr-specific history/rewrite/suggest (moved to plugin)"
```
---
## Task 9: Admin API rewrite
**Files:**
- Rewrite: `internal/api/handlers/autoscan.go`
- Modify: `internal/api/handlers/autoscan_test.go`
- [ ] **Step 1: Define the endpoints**
Rewrite the handler to back the new model. Endpoints (admin-gated, mounted in Task 10):
- `GET/PUT /admin/autoscan/settings`
- `GET/POST /admin/autoscan/connections`, `PUT/DELETE /admin/autoscan/connections/{id}`
- `GET /admin/autoscan/sources`, `PUT /admin/autoscan/sources/{id}` (enable, interval, connection binding)
- `POST /admin/autoscan/trigger` (async `PollOnce`, 202 — keep the detached-goroutine + channel-tested pattern from the closed PR's `autoscan_test.go`)
- `GET /admin/autoscan/status`
Strip credentials from responses (never emit `api_key_ref`/resolved keys), matching the `_SENSITIVE_METADATA_KEYS` defense-in-depth posture.
- [ ] **Step 2: Port + extend the handler tests**
Carry over the trigger test (the `done`-channel synchronization fix from `d3ee373`), the FK→404 test (now: unknown source/connection → 404), and a secrets-not-leaked test for the connections list.
- [ ] **Step 3: Run the handler tests (libvips container)**
Run (in the libvips-equipped container per Prerequisites #3):
`go test ./internal/api/handlers/ -run Autoscan`
Expected: PASS.
- [ ] **Step 4: Commit**
```bash
git add internal/api/handlers/autoscan.go internal/api/handlers/autoscan_test.go
git commit -m "feat(api): autoscan v2 admin endpoints (settings, connections, sources)"
```
---
## Task 10: Wiring + rate limits + retire old coupling
**Files:**
- Modify: `internal/api/router.go`
- Modify: `cmd/silo/main.go`
- [ ] **Step 1: Rebuild the service wiring**
In `internal/api/router.go` and `cmd/silo/main.go`, construct the v2 service:
```go
autoscanRepo := autoscan.NewRepository(deps.DB)
provider := autoscan.NewPluginProvider(pluginServiceScanSourceAdapter{pluginService})
connResolver := autoscan.NewConnectionResolver(requestIntegrationLookup{requestsRepo}, serverSettingsSecretResolver{settingsRepo})
autoscanSvc := autoscan.NewService(autoscanRepo, provider, connResolver, deps.Resolver, deps.ScanQueue, autoscan.NewRedisSuppressor(deps.RedisClient))
autoscanHandler = handlers.NewAutoscanHandler(autoscanRepo, autoscanSvc)
```
Replace the old route block with the Task 9 routes. Keep the `catalog_*`/admin rate-limit conventions consistent with the security standards (per-user/admin caps on the new endpoints; mirror the request-integrations admin tier).
- [ ] **Step 2: Retarget the poll task**
`internal/taskmanager/tasks/autoscan_poll.go` calls `Service.PollOnce` — unchanged behavior; confirm it compiles against the rebuilt service and uses `Settings.DefaultPollIntervalSeconds` for the interval (per-source overrides are honored inside `PollOnce`).
- [ ] **Step 3: Build the whole tree**
Run (libvips container): `go build ./...`
Expected: exits 0.
- [ ] **Step 4: Commit**
```bash
git add internal/api/router.go cmd/silo/main.go internal/taskmanager/tasks/autoscan_poll.go
git commit -m "feat(autoscan): wire v2 service, routes, and poll task"
```
---
## Task 11: Full sweep + finalize SDK dependency
**Files:**
- Modify: `go.mod`, `go.sum`
- [ ] **Step 1: Full build + test (libvips container)**
Run in the libvips-equipped container (Prerequisites #3):
```bash
go build ./... && go test ./... && go vet ./...
```
Expected: build OK; all packages `ok`; vet clean. `gofmt -l internal migrations` prints nothing.
- [ ] **Step 2: Finalize the SDK version (after `v0.5.0` is tagged upstream)**
Once `silo-plugin-sdk` PR #2 is merged and `v0.5.0` tagged:
```bash
go mod edit -dropreplace github.com/Silo-Server/silo-plugin-sdk
go get github.com/Silo-Server/silo-plugin-sdk@v0.5.0
go mod tidy
go build ./...
```
Expected: builds against the released tag with no replace directive.
If `v0.5.0` is not yet tagged when this plan is otherwise complete, leave the replace directive in place, commit it, and note in the PR that the dependency must be finalized before merge (do not merge a `replace` pointing at a local path).
- [ ] **Step 3: Commit**
```bash
git add go.mod go.sum
git commit -m "build: depend on silo-plugin-sdk v0.5.0 (scan_source.v1)"
```
---
## Self-Review notes
- **Spec coverage:** §7 engine (Tasks 6–7), §8 connections incl. reuse-or-own + soft Requests link (Tasks 3,5), §9 data model decoupled from Requests + Autoscan-owned state (Tasks 3–4,9–10), §5/§11 host consumes `scan_source.v1` (Tasks 1–2). §10 (arr plugin) and §9 UI are separate plans.
- **Salvage map honored (§13):** engine/dedupe/suppress kept and generalized (Tasks 6–8); arr history/rewrite/suggest deleted (Task 8); `last_poll_at` → opaque `marker` (Tasks 3–4,7).
- **Out of scope here:** the Autoscan admin **UI** category (Part 2 plan) and the **arr plugin** (separate repo plan). This backend is testable on its own via the repository/engine/handler tests.
- **Risk carried from spec §14:** plugin multiplicity (one installation per arr server vs. many connections per installation) is modeled as one `autoscan_sources` row per `(installation_id, capability_id)`; if the runtime supports multiple capability instances per installation this maps cleanly, otherwise it implies one installation per server. Confirm during Task 1–2 by inspecting `ensureClient`/installation handling.
- **Type consistency:** `ResolvedConnection{BaseURL,APIKey}`, `Connection{BaseURL,APIKeyRef,RequestIntegrationID}`, `Source{InstallationID,CapabilityID,ConnectionID,Marker}`, `ScanSourceProvider.PollChanges(...)`, `Service.PollOnce` used consistently across Tasks 4–10.
docs/superpowers/plans/2026-06-02-autoscan-rewrite-sync.md
+694
View File
@@ -0,0 +1,694 @@
# Autoscan Rewrite-Sync Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** A "Sync rewrites from arr" action that suggests autoscan `path_rewrites` for an instance by matching its arr root folders to Silo media folders on shared trailing path segments, previewed and committed by the admin.
**Architecture:** A pure `suggestRewrites` matcher in `internal/autoscan`, a `Service.SuggestRewrites` method backed by two new injected deps (an arr root-folder client and a Silo folder lister), one read-only admin endpoint returning a preview, and a "Sync from arr" button in the existing Autoscan tab that merges proposals into the editor (admin saves via the existing per-source PUT). Manual rewrites stay the source of truth.
**Tech Stack:** Go (standard `testing`), React/TypeScript.
**Spec:** `docs/superpowers/specs/2026-06-02-autoscan-rewrite-sync-design.md`
**Commands assume the repository root is the cwd.** Go tests: `go test ./internal/autoscan/...`. Frontend: `cd web && pnpm exec tsc -b && pnpm run lint`. Ensure the Go toolchain is on `PATH` (prepend its `bin` directory if `go` is missing). `internal/api`/`internal/api/handlers` cannot be compiled without libvips; verify those via `gofmt` + the Docker build.
---
## Phase 0 — Branch
- [ ] **Step 0.1: Confirm branch**
```bash
git rev-parse --abbrev-ref HEAD # expect: feat/autoscan-arr-polling
```
This feature extends the autoscan branch. If you're elsewhere: `git checkout feat/autoscan-arr-polling`.
---
## Phase 1 — Core matcher (pure, TDD)
### Task 1: suggestRewrites + types
**Files:**
- Create: `internal/autoscan/suggest.go`
- Create: `internal/autoscan/suggest_test.go`
- [ ] **Step 1.1: Write the failing test**
`internal/autoscan/suggest_test.go`:
```go
package autoscan
import (
"reflect"
"testing"
)
func TestSuggestRewrites(t *testing.T) {
silo := []string{
"/mnt/media/happy/storage2/tvshows1",
"/mnt/media/happy4k/4ktv7",
"/mnt/media/storage/Anime/Subs",
"/mnt/media/storage/Anime2/Subs",
"/library/Films",
"/tank/television/Show",
}
t.Run("multi-segment unique match", func(t *testing.T) {
got := suggestRewrites([]string{"/mnt/happy/storage2/tvshows1"}, silo, nil)
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" || got.Proposed[0].MatchDepth != 2 {
t.Fatalf("proposed=%+v", got.Proposed)
}
})
t.Run("single-segment unique match (different parents)", func(t *testing.T) {
got := suggestRewrites([]string{"/mnt/kodama/storage2/4ktv7"}, silo, nil)
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy4k/4ktv7" || got.Proposed[0].MatchDepth != 1 {
t.Fatalf("proposed=%+v", got.Proposed)
}
})
t.Run("longest-suffix disambiguation", func(t *testing.T) {
got := suggestRewrites([]string{"/mnt/kodama/storage1/Anime/Subs"}, silo, nil)
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/storage/Anime/Subs" || got.Proposed[0].MatchDepth != 2 {
t.Fatalf("expected Anime/Subs (depth 2), got %+v", got.Proposed)
}
if len(got.Ambiguous) != 0 {
t.Fatalf("should not be ambiguous: %+v", got.Ambiguous)
}
})
t.Run("unmatched when no shared segment", func(t *testing.T) {
got := suggestRewrites([]string{"/data/Movies"}, silo, nil)
if len(got.Unmatched) != 1 || got.Unmatched[0] != "/data/Movies" || len(got.Proposed) != 0 {
t.Fatalf("got=%+v", got)
}
})
t.Run("leaf match across unlike layouts", func(t *testing.T) {
got := suggestRewrites([]string{"/srv/tv/Show"}, silo, nil)
if len(got.Proposed) != 1 || got.Proposed[0].To != "/tank/television/Show" {
t.Fatalf("got=%+v", got)
}
})
t.Run("ambiguous tie", func(t *testing.T) {
// a root whose only shared segment is "Subs" ties two folders
got := suggestRewrites([]string{"/foo/bar/Subs"}, silo, nil)
if len(got.Ambiguous) != 1 || len(got.Ambiguous[0].Candidates) != 2 {
t.Fatalf("expected ambiguous with 2 candidates, got %+v", got.Ambiguous)
}
})
t.Run("covered by existing rule", func(t *testing.T) {
existing := []PathRewrite{{From: "/mnt/happy", To: "/mnt/media/happy"}}
got := suggestRewrites([]string{"/mnt/happy/storage2/tvshows1"}, silo, existing)
if len(got.Covered) != 1 || len(got.Proposed) != 0 {
t.Fatalf("expected covered, got %+v", got)
}
})
t.Run("normalization: trailing slash, backslashes, dup slashes", func(t *testing.T) {
got := suggestRewrites([]string{`\mnt\happy\\storage2\tvshows1\`}, silo, nil)
if len(got.Proposed) != 1 || got.Proposed[0].From != "/mnt/happy/storage2/tvshows1" || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" {
t.Fatalf("normalization failed: %+v", got.Proposed)
}
})
}
func TestCommonSuffixLen(t *testing.T) {
cases := []struct {
a, b []string
want int
}{
{[]string{"a", "b", "c"}, []string{"x", "b", "c"}, 2},
{[]string{"4ktv7"}, []string{"happy4k", "4ktv7"}, 1},
{[]string{"4ktv7"}, []string{"4ktv70"}, 0}, // full-segment, not substring
{[]string{"a"}, []string{"b"}, 0},
}
for _, tc := range cases {
if got := commonSuffixLen(tc.a, tc.b); got != tc.want {
t.Fatalf("commonSuffixLen(%v,%v)=%d want %d", tc.a, tc.b, got, tc.want)
}
}
}
```
- [ ] **Step 1.2: Run it — expect failure**
```bash
go test ./internal/autoscan/ -run 'TestSuggestRewrites|TestCommonSuffixLen' -v
```
Expected: FAIL (`suggestRewrites` undefined).
- [ ] **Step 1.3: Implement** `internal/autoscan/suggest.go`:
```go
package autoscan
import "strings"
// RewriteSuggestions is the result of matching arr root folders to Silo folders.
type RewriteSuggestions struct {
Proposed []ProposedRewrite `json:"proposed"`
Unmatched []string `json:"unmatched"`
Ambiguous []AmbiguousRoot `json:"ambiguous"`
Covered []string `json:"covered"`
}
// ProposedRewrite is a suggested rewrite plus its confidence (shared trailing segments).
type ProposedRewrite struct {
From string `json:"from"`
To string `json:"to"`
MatchDepth int `json:"match_depth"`
}
// AmbiguousRoot is an arr root that tied across multiple Silo folders.
type AmbiguousRoot struct {
Root string `json:"root"`
Candidates []string `json:"candidates"`
}
// normalizePath makes a path comparable: backslashes -> '/', collapse duplicate
// slashes, strip a trailing slash (but keep a bare "/").
func normalizePath(p string) string {
p = normalizeSeparators(strings.TrimSpace(p))
for strings.Contains(p, "//") {
p = strings.ReplaceAll(p, "//", "/")
}
if len(p) > 1 {
p = strings.TrimRight(p, "/")
}
return p
}
func segments(p string) []string {
p = strings.Trim(p, "/")
if p == "" {
return nil
}
return strings.Split(p, "/")
}
// commonSuffixLen counts equal trailing segments (full-segment equality).
func commonSuffixLen(a, b []string) int {
i, j, n := len(a)-1, len(b)-1, 0
for i >= 0 && j >= 0 && a[i] == b[j] {
n++
i--
j--
}
return n
}
// coveredBy reports whether an existing rewrite already matches root (same
// boundary rule as applyRewrites).
func coveredBy(root string, existing []PathRewrite) bool {
for _, rw := range existing {
from := strings.TrimRight(strings.TrimSpace(rw.From), "/")
if from == "" {
continue
}
if root == from || strings.HasPrefix(root, from+"/") {
return true
}
}
return false
}
// suggestRewrites matches each arr root to the Silo folder sharing the most
// trailing path segments (unique winner). Roots already handled by an existing
// rewrite are reported as Covered; roots with no shared segment as Unmatched;
// ties as Ambiguous. Pure: no I/O, no deployment constants.
func suggestRewrites(arrRoots, siloFolderPaths []string, existing []PathRewrite) RewriteSuggestions {
siloNorm := make([]string, 0, len(siloFolderPaths))
siloSegs := make([][]string, 0, len(siloFolderPaths))
for _, p := range siloFolderPaths {
n := normalizePath(p)
if n == "" {
continue
}
siloNorm = append(siloNorm, n)
siloSegs = append(siloSegs, segments(n))
}
var out RewriteSuggestions
for _, raw := range arrRoots {
root := normalizePath(raw)
if root == "" {
continue
}
if coveredBy(root, existing) {
out.Covered = append(out.Covered, root)
continue
}
rootSegs := segments(root)
best := 0
var winners []string
for i, segs := range siloSegs {
n := commonSuffixLen(rootSegs, segs)
if n == 0 {
continue
}
if n > best {
best, winners = n, []string{siloNorm[i]}
} else if n == best {
winners = append(winners, siloNorm[i])
}
}
switch {
case best == 0:
out.Unmatched = append(out.Unmatched, root)
case len(winners) == 1:
out.Proposed = append(out.Proposed, ProposedRewrite{From: root, To: winners[0], MatchDepth: best})
default:
out.Ambiguous = append(out.Ambiguous, AmbiguousRoot{Root: root, Candidates: winners})
}
}
return out
}
```
- [ ] **Step 1.4: Run it — expect pass; commit**
```bash
go test ./internal/autoscan/ -run 'TestSuggestRewrites|TestCommonSuffixLen' -v
gofmt -l internal/autoscan/suggest.go internal/autoscan/suggest_test.go
git add internal/autoscan/suggest.go internal/autoscan/suggest_test.go
git commit -m "feat(autoscan): suffix-match rewrite suggester"
```
---
## Phase 2 — Service support
### Task 2: Repository.GetSource
**Files:**
- Modify: `internal/autoscan/repository.go`
- [ ] **Step 2.1: Implement** `GetSource` (single-source join, includes disabled). Add after `ListEnabledSources`:
```go
// GetSource returns one instance's autoscan state joined with its
// request_integrations row (kind/base_url/api_key_ref), regardless of enabled.
func (r *Repository) GetSource(ctx context.Context, integrationID string) (*Source, error) {
row := r.pool.QueryRow(ctx, sourceSelect+` WHERE ri.id = $1`, integrationID)
src, err := scanSource(row)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, fmt.Errorf("%w: %s", ErrIntegrationNotFound, integrationID)
}
return nil, fmt.Errorf("get autoscan source: %w", err)
}
return &src, nil
}
```
(`errors`, `pgx`, `fmt`, `ErrIntegrationNotFound`, and `sourceSelect`/`scanSource` already exist in the file.)
- [ ] **Step 2.2: Build + commit**
```bash
go build ./internal/autoscan/... && go test ./internal/autoscan/...
git add internal/autoscan/repository.go
git commit -m "feat(autoscan): GetSource single-source lookup"
```
### Task 3: RootFolderClient + FolderLister adapters
**Files:**
- Create: `internal/autoscan/suggest_deps.go`
- [ ] **Step 3.1: Implement the two adapters + interfaces**
```go
package autoscan
import (
"context"
"net/http"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/requests/arrclient"
)
// RootFolderClient lists a Radarr/Sonarr instance's configured root folder paths.
type RootFolderClient interface {
RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error)
}
// FolderLister lists every Silo media-folder path.
type FolderLister interface {
ListFolderPaths(ctx context.Context) ([]string, error)
}
type arrRootFolderClient struct{ httpClient *http.Client }
// NewArrRootFolderClient returns a RootFolderClient backed by the shared arrclient.
func NewArrRootFolderClient(httpClient *http.Client) RootFolderClient {
return &arrRootFolderClient{httpClient: httpClient}
}
func (c *arrRootFolderClient) RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error) {
client := arrclient.New(baseURL, apiKey, c.httpClient)
folders, err := arrclient.ListRootFolders(ctx, client)
if err != nil {
return nil, err
}
paths := make([]string, 0, len(folders))
for _, f := range folders {
if f.Path != "" {
paths = append(paths, f.Path)
}
}
return paths, nil
}
type catalogFolderLister struct{ repo *catalog.FolderRepository }
// NewCatalogFolderLister adapts catalog.FolderRepository to FolderLister.
func NewCatalogFolderLister(repo *catalog.FolderRepository) FolderLister {
return &catalogFolderLister{repo: repo}
}
func (l *catalogFolderLister) ListFolderPaths(ctx context.Context) ([]string, error) {
folders, err := l.repo.List(ctx)
if err != nil {
return nil, err
}
var paths []string
for _, f := range folders {
paths = append(paths, f.Paths...)
}
return paths, nil
}
```
Note: `autoscan` already depends on `catalog` (via `scantrigger`) and on `requests/arrclient` (via `history.go`), so these imports add no new module-level coupling and no cycle (`catalog` does not import `autoscan`).
- [ ] **Step 3.2: Build + commit**
```bash
go build ./internal/autoscan/...
git add internal/autoscan/suggest_deps.go
git commit -m "feat(autoscan): arr root-folder client + Silo folder lister"
```
### Task 4: Service.SuggestRewrites
**Files:**
- Modify: `internal/autoscan/service.go`
- Modify: `internal/autoscan/service_test.go`
- [ ] **Step 4.1: Write the failing test** (append to `service_test.go`):
```go
type fakeRootFolders struct {
paths []string
err error
}
func (f fakeRootFolders) RootFolders(context.Context, string, string) ([]string, error) {
return f.paths, f.err
}
type fakeFolderLister struct{ paths []string }
func (f fakeFolderLister) ListFolderPaths(context.Context) ([]string, error) { return f.paths, nil }
type sourceGetterStore struct {
fakeStore
src *Source
}
func (s *sourceGetterStore) GetSource(context.Context, string) (*Source, error) { return s.src, nil }
func TestSuggestRewritesService(t *testing.T) {
store := &sourceGetterStore{src: &Source{IntegrationID: "i1", Kind: "sonarr", BaseURL: "http://x", APIKeyRef: "k"}}
svc := NewService(store, &fakeHistory{}, fakeResolver{}, &recordingQueuer{}, allowSuppressor{}, nil)
svc.SetRewriteResolvers(
fakeRootFolders{paths: []string{"/mnt/happy/storage2/tvshows1", "/data/Movies"}},
fakeFolderLister{paths: []string{"/mnt/media/happy/storage2/tvshows1"}},
)
got, err := svc.SuggestRewrites(context.Background(), "i1")
if err != nil {
t.Fatalf("SuggestRewrites: %v", err)
}
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" {
t.Fatalf("proposed=%+v", got.Proposed)
}
if len(got.Unmatched) != 1 || got.Unmatched[0] != "/data/Movies" {
t.Fatalf("unmatched=%+v", got.Unmatched)
}
}
```
> The `Store` interface needs a `GetSource` method so the service can call it. Add `GetSource(ctx context.Context, integrationID string) (*Source, error)` to the `Store` interface in `service.go`. `*Repository` already implements it (Task 2). The test's `sourceGetterStore` embeds the existing `fakeStore` and adds `GetSource`. If `fakeStore` does not satisfy the widened interface on its own, that's fine — only `sourceGetterStore` is used here; but ensure the package still compiles (other tests construct `fakeStore` directly and pass it where `Store` is required — add a `GetSource` method to `fakeStore` returning `(nil, nil)` so it still satisfies `Store`).
- [ ] **Step 4.2: Run it — expect failure**
```bash
go test ./internal/autoscan/ -run TestSuggestRewritesService -v
```
Expected: FAIL (`SetRewriteResolvers`/`SuggestRewrites` undefined).
- [ ] **Step 4.3: Implement** in `service.go`. Add fields + setter + method, and add `GetSource` to `Store`:
```go
// add to the Store interface:
// GetSource(ctx context.Context, integrationID string) (*Source, error)
// add fields to Service struct:
// rootFolders RootFolderClient
// folders FolderLister
// SetRewriteResolvers wires the deps used by SuggestRewrites (optional; only the
// admin-facing service needs them).
func (s *Service) SetRewriteResolvers(rootFolders RootFolderClient, folders FolderLister) {
s.rootFolders = rootFolders
s.folders = folders
}
// SuggestRewrites matches an instance's arr root folders to Silo media folders.
func (s *Service) SuggestRewrites(ctx context.Context, integrationID string) (RewriteSuggestions, error) {
if s.rootFolders == nil || s.folders == nil {
return RewriteSuggestions{}, fmt.Errorf("autoscan: rewrite suggestion not configured")
}
src, err := s.store.GetSource(ctx, integrationID)
if err != nil {
return RewriteSuggestions{}, err
}
apiKey := src.APIKeyRef
if s.secrets != nil && apiKey != "" {
if resolved, rerr := s.secrets.Get(ctx, apiKey); rerr == nil && resolved != "" {
apiKey = resolved
}
}
arrRoots, err := s.rootFolders.RootFolders(ctx, src.BaseURL, apiKey)
if err != nil {
return RewriteSuggestions{}, fmt.Errorf("autoscan: list arr root folders: %w", err)
}
siloPaths, err := s.folders.ListFolderPaths(ctx)
if err != nil {
return RewriteSuggestions{}, fmt.Errorf("autoscan: list silo folders: %w", err)
}
return suggestRewrites(arrRoots, siloPaths, src.PathRewrites), nil
}
```
Add `GetSource` to `fakeStore` in `service_test.go` (`func (f *fakeStore) GetSource(context.Context, string) (*Source, error) { return nil, nil }`).
- [ ] **Step 4.4: Run tests — expect pass; commit**
```bash
go test ./internal/autoscan/...
gofmt -l internal/autoscan/service.go internal/autoscan/service_test.go
git add internal/autoscan/service.go internal/autoscan/service_test.go
git commit -m "feat(autoscan): Service.SuggestRewrites"
```
---
## Phase 3 — API endpoint
### Task 5: Handler + route
**Files:**
- Modify: `internal/api/handlers/autoscan.go`
- Modify: `internal/api/router.go`
- Modify: `internal/api/handlers/autoscan_test.go`
- [ ] **Step 5.1: Extend the handler's service interface + add the handler method**
In `autoscan.go`, add `SuggestRewrites` to the `autoscanTriggerer` interface:
```go
type autoscanTriggerer interface {
PollOnce(ctx context.Context) error
SuggestRewrites(ctx context.Context, integrationID string) (autoscan.RewriteSuggestions, error)
}
```
Add the handler:
```go
func (h *AutoscanHandler) HandleRewriteSuggestions(w http.ResponseWriter, r *http.Request) {
id := strings.TrimSpace(chi.URLParam(r, "id"))
suggestions, err := h.svc.SuggestRewrites(r.Context(), id)
if err != nil {
if errors.Is(err, autoscan.ErrIntegrationNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Autoscan source not found")
return
}
// arr unreachable / bad key / folder list failure
writeError(w, http.StatusBadGateway, "arr_unreachable", "Could not load root folders from the arr instance")
return
}
writeJSON(w, http.StatusOK, suggestions)
}
```
(`errors`, `autoscan`, `chi`, `strings`, `writeError`, `writeJSON` already imported.)
- [ ] **Step 5.2: Mount the route** in `router.go`, inside the autoscan route block:
```go
r.Get("/autoscan/sources/{id}/rewrite-suggestions", autoscanHandler.HandleRewriteSuggestions)
```
- [ ] **Step 5.3: Wire the service deps** in `router.go` where `autoscanSvc` is built (after `autoscan.NewService(...)`):
```go
autoscanSvc.SetRewriteResolvers(
autoscan.NewArrRootFolderClient(nil),
autoscan.NewCatalogFolderLister(deps.FolderRepo),
)
```
(This is inside the existing `if deps.FolderRepo != nil && deps.LibraryScanQueue != nil` block, so `deps.FolderRepo` is non-nil here.)
- [ ] **Step 5.4: Update the handler test fake**
In `autoscan_test.go`, the fake satisfying `autoscanTriggerer` (e.g. `fakeAutoscanTriggerer`) gains:
```go
func (f *fakeAutoscanTriggerer) SuggestRewrites(context.Context, string) (autoscan.RewriteSuggestions, error) {
return autoscan.RewriteSuggestions{}, nil
}
```
Add a test asserting `HandleRewriteSuggestions` returns 200 with JSON for a known id (mirror the existing handler tests).
- [ ] **Step 5.5: Verify (limited) + commit**
```bash
# ensure the Go toolchain is on PATH
go vet ./internal/autoscan/...
gofmt -l internal/api/handlers/autoscan.go internal/api/router.go internal/api/handlers/autoscan_test.go
# internal/api cannot compile here (libvips) — Docker verifies later.
git add internal/api/handlers/autoscan.go internal/api/router.go internal/api/handlers/autoscan_test.go
git commit -m "feat(autoscan): rewrite-suggestions endpoint"
```
---
## Phase 4 — Frontend
### Task 6: Types + hook
**Files:**
- Modify: `web/src/api/types.ts`
- Modify: `web/src/hooks/queries/useAutoscan.ts`
- [ ] **Step 6.1: Types** — add to `types.ts`:
```ts
export interface AutoscanProposedRewrite {
from: string;
to: string;
match_depth: number;
}
export interface AutoscanAmbiguousRoot {
root: string;
candidates: string[];
}
export interface AutoscanRewriteSuggestions {
proposed: AutoscanProposedRewrite[];
unmatched: string[];
ambiguous: AutoscanAmbiguousRoot[];
covered: string[];
}
```
- [ ] **Step 6.2: Hook** — add to `useAutoscan.ts` a mutation that fetches suggestions for an id:
```ts
export function useAutoscanRewriteSuggestions() {
return useMutation({
mutationFn: (id: string) =>
api<AutoscanRewriteSuggestions>(`/admin/autoscan/sources/${id}/rewrite-suggestions`),
onError: (err) =>
toast.error(err instanceof Error ? err.message : "Could not sync rewrites from the arr instance"),
});
}
```
Match the import/`api`/`toast`/`useMutation` conventions already in the file (it imports `api` and `toast` already; reuse them; import the new type).
- [ ] **Step 6.3: Lint + commit**
```bash
cd web && pnpm exec tsc -b && pnpm run lint && pnpm exec prettier --check src/api/types.ts src/hooks/queries/useAutoscan.ts
git add web/src/api/types.ts web/src/hooks/queries/useAutoscan.ts
git commit -m "feat(web): autoscan rewrite-suggestions types and hook"
```
### Task 7: Sync button + preview in AutoscanSourceEditor
**Files:**
- Modify: `web/src/pages/AdminRequests.tsx`
- [ ] **Step 7.1: Add the button + preview**
In `AutoscanSourceEditor`:
- Call `const suggest = useAutoscanRewriteSuggestions();` and add local state `const [preview, setPreview] = useState<AutoscanRewriteSuggestions | null>(null);` and `const [selected, setSelected] = useState<Set<string>>(new Set());`.
- Add a **"Sync from arr"** `Button` beside Save: `onClick={async () => { const s = await suggest.mutateAsync(source.integration_id); setPreview(s); setSelected(new Set(s.proposed.map((p) => p.from))); }}` (disabled while `suggest.isPending`).
- When `preview` is set, render a panel (reuse the file's `Dialog` or an inline bordered panel) with three sections:
- **Proposed:** each `proposed[]` row — a checkbox bound to `selected.has(p.from)` (toggle updates the set), `from → to`, and a `Badge` showing confidence: `match_depth >= 2 ? "${match_depth} segments" : "1 segment — weak"` (destructive/secondary variant for weak).
- **Unmatched:** muted list of `unmatched[]` ("no Silo match — add manually if needed").
- **Ambiguous:** list of `ambiguous[]` (`root` + its `candidates` joined) — info only.
- An **"Add selected to rewrites"** button that merges the checked proposals into the editor's existing rewrites state, deduped by `from` (skip any whose `from` already exists), then `setPreview(null)`. Do NOT auto-save — the admin then clicks the existing **Save**.
- Keep the existing manual path-rewrite editor unchanged; sync only appends rows to it.
Reuse existing `Button`/`Badge`/`Field`/`Dialog` primitives and the editor's existing rewrites-state setter.
- [ ] **Step 7.2: Lint + commit**
```bash
cd web && pnpm exec tsc -b && pnpm run lint && pnpm exec prettier --check src/pages/AdminRequests.tsx
git add web/src/pages/AdminRequests.tsx
git commit -m "feat(web): autoscan sync-rewrites preview"
```
---
## Phase 5 — Verification
### Task 8: Full verification
- [ ] **Step 8.1: Go**
```bash
# ensure the Go toolchain is on PATH
go test ./internal/autoscan/... 2>&1 | tail
go vet ./internal/autoscan/...
gofmt -l internal/autoscan internal/api/handlers/autoscan.go internal/api/router.go
```
Expected: all pass; vet/gofmt clean.
- [ ] **Step 8.2: Frontend**
```bash
cd web && pnpm exec tsc -b && pnpm run lint && pnpm run format:check
```
- [ ] **Step 8.3: Docker build (closes the libvips gap)**
```bash
docker build --build-arg BUILD_REVISION=$(git rev-parse HEAD) --build-arg BUILD_DIRTY=false -t silo-server:autoscan .
```
Expected: exit 0 (handlers + router + frontend compile).
- [ ] **Step 8.4: Smoke (optional)**
On a deploy with arr instances configured: open an instance in the Autoscan tab, click **Sync from arr**, confirm the preview shows proposed (with confidence) + unmatched + ambiguous, "Add selected" appends deduped rows, and Save persists them.
---
## Self-review notes (resolved)
- **Spec §1 matcher** → Task 1 (incl. normalization, covered-first, suffix depth, unlike-deployment fixtures). **§2 endpoint/service** → Tasks 2–5 (`GetSource`, `RootFolderClient`/`FolderLister`, `SuggestRewrites`, endpoint, 404/502 mapping). **§3 frontend** → Tasks 6–7. **§4 testing** → Tasks 1,4,5 + Task 8.
- **Manual-preserving**: sync only *appends* deduped rows to the editor; the admin saves via the existing `PUT` (Task 7). Never auto-writes.
- **Generic**: `suggestRewrites` takes plain string slices, no deployment constants; tests include `/data/Movies` and `/srv/tv` fixtures (Task 1).
- **Optional deps**: `SetRewriteResolvers` keeps `main.go`'s poll service untouched; `SuggestRewrites` errors if unset (Task 4).
- **Type consistency**: `RewriteSuggestions`/`ProposedRewrite`/`AmbiguousRoot` (Go) ↔ `AutoscanRewriteSuggestions`/`AutoscanProposedRewrite`/`AutoscanAmbiguousRoot` (TS); `match_depth` JSON tag matches.
docs/superpowers/plans/2026-06-02-autoscan-scan-source-sdk-capability.md
+375
View File
@@ -0,0 +1,375 @@
# `scan_source.v1` SDK Capability — Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Add a new, additive `scan_source.v1` plugin capability to `silo-plugin-sdk` so the Silo host can pull changed paths from a scan-source plugin on a timer.
**Architecture:** Mirror the existing `scheduled_task.v1` capability exactly — a single-RPC, request/response gRPC service defined in a `.proto`, generated into `pkg/pluginproto`, then registered in three Go helper sites (`capability.KnownTypes`, the `runtime.CapabilityServers` struct, and the `runtime.GRPCPlugin` server registration). Nothing existing is modified in shape; only new symbols are added. Tag a new SDK release so the host and the arr plugin can build against it.
**Tech Stack:** Go, Protocol Buffers (proto3), `buf` codegen (`buf.gen.yaml`), `google.golang.org/protobuf`, `google.golang.org/grpc`, the `hashicorp/go-plugin` runtime the SDK wraps.
---
## Prerequisites (read before starting)
This plan targets the **`silo-plugin-sdk`** repository (Go module `github.com/Silo-Server/silo-plugin-sdk`), which is **not currently checked out** in this workspace — only `silo-server` and `silo-plugin-tmdb` are. Before executing:
1. Clone `silo-plugin-sdk` as a writable working copy beside the other repos. Commands below assume **the silo-plugin-sdk repository root is the cwd**.
2. Ensure the proto toolchain is available: the `make proto` target auto-installs `buf`, `protoc-gen-go@v1.36.11`, and `protoc-gen-go-grpc@v1.6.1` into `$(GO_BIN)`, but it requires the **`protoc`** binary to already be on `PATH` (it errors `protoc is required` otherwise). Install `protoc` if absent.
3. This SDK is the **long pole**: the silo-server host plan and the arr-plugin plan both depend on the version tagged at the end of this plan (Task 6). Do not start those until this is tagged.
**Pattern reference (current `scheduled_task.v1`, the template this mirrors):**
- Proto: `proto/silo/plugin/v1/scheduled_task.proto`
- Generated: `pkg/pluginproto/silo/plugin/v1/scheduled_task.pb.go` + `scheduled_task_grpc.pb.go`
- Capability const + allowlist: `pkg/pluginsdk/capability/capability.go`
- Runtime wiring: `pkg/pluginsdk/runtime/runtime.go` (struct field, `Client` accessor, `GRPCServer` registration)
- Manifest acceptance test pattern: `pkg/pluginsdk/manifest/manifest_test.go`
---
## File Structure
| File | Responsibility | Action |
|---|---|---|
| `proto/silo/plugin/v1/scan_source.proto` | The `ScanSource` service + `PollChanges` request/response messages | Create |
| `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go` | Generated message types | Create (codegen) |
| `pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go` | Generated client/server stubs | Create (codegen) |
| `pkg/pluginsdk/capability/capability.go` | `ScanSource` const + add to `KnownTypes` allowlist | Modify |
| `pkg/pluginsdk/capability/capability_test.go` | Assert `ScanSource` is a known type | Create or Modify |
| `pkg/pluginsdk/runtime/runtime.go` | `CapabilityServers.ScanSource` field, `Client.ScanSource()` accessor, `GRPCServer` registration | Modify |
| `pkg/pluginsdk/runtime/runtime_test.go` | Assert a `ScanSource` server registers without error | Create or Modify |
| `pkg/pluginsdk/manifest/manifest_test.go` | Assert a manifest declaring `scan_source.v1` loads | Modify |
---
## Task 1: Define and generate the `scan_source.v1` proto
**Files:**
- Create: `proto/silo/plugin/v1/scan_source.proto`
- Create (via codegen): `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go`, `pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go`
- [ ] **Step 1: Write the proto**
Create `proto/silo/plugin/v1/scan_source.proto` (the `go_package` option must match the sibling protos exactly so generated code lands in package `pluginv1`):
```proto
syntax = "proto3";
package silo.plugin.v1;
option go_package = "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1;pluginv1";
// ScanSource lets the host pull changed filesystem paths from a provider
// (e.g. Sonarr/Radarr, inotify, Ceph). Pull only: the host owns the timer and
// calls PollChanges; the plugin never initiates.
service ScanSource {
rpc PollChanges(PollChangesRequest) returns (PollChangesResponse);
}
message PollChangesRequest {
// Which configured scan_source capability instance is being polled.
string capability_id = 1;
// Opaque continuation token from the previous PollChanges. Empty on first
// run, which the provider treats as "start from now" (do not replay history).
string marker = 2;
}
message PollChangesResponse {
// Absolute paths already translated into Silo's filesystem namespace (the
// plugin has applied its own path rewrites). Files or directories.
repeated string changed_paths = 1;
// Opaque continuation token. The host stores it verbatim and echoes it back
// on the next PollChanges; the host never parses it.
string next_marker = 2;
}
```
- [ ] **Step 2: Generate the Go code**
Run: `make proto`
Expected: exits 0; creates `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go` and `scan_source_grpc.pb.go`. (If it prints `protoc is required`, install `protoc` and re-run.)
- [ ] **Step 3: Verify the generated symbols exist and compile**
Run: `go build ./... && grep -l "func NewScanSourceClient" pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go && grep -l "RegisterScanSourceServer" pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go`
Expected: build succeeds; both `grep`s print the filename (confirming `ScanSourceClient`, `ScanSourceServer`, `NewScanSourceClient`, `RegisterScanSourceServer` were generated — `require_unimplemented_servers=false` in `buf.gen.yaml` means no forced `UnimplementedScanSourceServer` embedding).
- [ ] **Step 4: Commit**
```bash
git add proto/silo/plugin/v1/scan_source.proto pkg/pluginproto/silo/plugin/v1/scan_source.pb.go pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go
git commit -m "feat(proto): add scan_source.v1 capability service"
```
---
## Task 2: Register the capability type in the allowlist
The manifest loader rejects capability types not in `capability.KnownTypes` (see `TestLoadRejectsUnknownCapabilityType`). A manifest declaring `scan_source.v1` cannot load until the type is registered here.
**Files:**
- Modify: `pkg/pluginsdk/capability/capability.go`
- Create or Modify: `pkg/pluginsdk/capability/capability_test.go`
- [ ] **Step 1: Write the failing test**
Append to `pkg/pluginsdk/capability/capability_test.go` (create the file with this `package capability` header if it does not exist):
```go
package capability
import "testing"
func TestScanSourceIsKnownType(t *testing.T) {
if ScanSource != "scan_source.v1" {
t.Fatalf("ScanSource const = %q, want %q", ScanSource, "scan_source.v1")
}
found := false
for _, k := range KnownTypes {
if k == ScanSource {
found = true
break
}
}
if !found {
t.Fatalf("ScanSource (%q) missing from KnownTypes %v", ScanSource, KnownTypes)
}
}
```
- [ ] **Step 2: Run test to verify it fails**
Run: `go test ./pkg/pluginsdk/capability/ -run TestScanSourceIsKnownType -v`
Expected: FAIL — `undefined: ScanSource` (the const does not exist yet).
- [ ] **Step 3: Add the const and allowlist entry**
In `pkg/pluginsdk/capability/capability.go`, add the const inside the existing `const (...)` block (after `EbookBackend`):
```go
EbookBackend = "ebook_backend.v1"
ScanSource = "scan_source.v1"
```
and add it to the `KnownTypes` slice (append `ScanSource` to the existing list):
```go
ScanSource,
```
- [ ] **Step 4: Run test to verify it passes**
Run: `go test ./pkg/pluginsdk/capability/ -run TestScanSourceIsKnownType -v`
Expected: PASS.
- [ ] **Step 5: Commit**
```bash
git add pkg/pluginsdk/capability/capability.go pkg/pluginsdk/capability/capability_test.go
git commit -m "feat(capability): register scan_source.v1 as a known capability type"
```
---
## Task 3: Wire the capability into the runtime
Add the `ScanSource` server to `CapabilityServers`, a `Client.ScanSource()` accessor, and the conditional registration in `GRPCServer` — mirroring `ScheduledTask` in `pkg/pluginsdk/runtime/runtime.go`.
**Files:**
- Modify: `pkg/pluginsdk/runtime/runtime.go`
- Create or Modify: `pkg/pluginsdk/runtime/runtime_test.go`
- [ ] **Step 1: Write the failing test**
Append to `pkg/pluginsdk/runtime/runtime_test.go` (create with the `package runtime` header if absent). This test constructs a real `*grpc.Server`, supplies a minimal `ScanSource` server, and asserts `GRPCServer` registers it without error:
```go
package runtime
import (
"context"
"testing"
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
"google.golang.org/grpc"
)
type stubScanSource struct{}
func (stubScanSource) PollChanges(context.Context, *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
return &pluginv1.PollChangesResponse{}, nil
}
func TestGRPCServerRegistersScanSource(t *testing.T) {
p := &GRPCPlugin{Servers: CapabilityServers{
Runtime: stubRuntime{}, // existing stub used by sibling runtime tests
ScanSource: stubScanSource{},
}}
srv := grpc.NewServer()
if err := p.GRPCServer(nil, srv); err != nil {
t.Fatalf("GRPCServer with ScanSource = %v, want nil", err)
}
if _, ok := srv.GetServiceInfo()["silo.plugin.v1.ScanSource"]; !ok {
t.Fatalf("ScanSource service not registered; got %v", srv.GetServiceInfo())
}
}
```
Note: `GRPCServer` requires `Servers.Runtime` to be non-nil (it errors otherwise). Reuse the existing runtime stub the sibling tests use. If no such stub exists in `runtime_test.go`, add a minimal one:
```go
type stubRuntime struct{ pluginv1.RuntimeServer }
```
- [ ] **Step 2: Run test to verify it fails**
Run: `go test ./pkg/pluginsdk/runtime/ -run TestGRPCServerRegistersScanSource -v`
Expected: FAIL — `unknown field 'ScanSource' in struct literal of type CapabilityServers` (the field does not exist yet).
- [ ] **Step 3: Add the struct field**
In `pkg/pluginsdk/runtime/runtime.go`, add to the `CapabilityServers` struct (after the `ScheduledTask` field):
```go
ScheduledTask pluginv1.ScheduledTaskServer
ScanSource pluginv1.ScanSourceServer
```
- [ ] **Step 4: Add the `Client` accessor**
After the existing `func (c *Client) ScheduledTask() ...` method, add:
```go
func (c *Client) ScanSource() pluginv1.ScanSourceClient {
return pluginv1.NewScanSourceClient(c.conn)
}
```
- [ ] **Step 5: Add the conditional registration in `GRPCServer`**
In `func (p *GRPCPlugin) GRPCServer(...)`, after the `ScheduledTask` registration block, add:
```go
if p.Servers.ScheduledTask != nil {
pluginv1.RegisterScheduledTaskServer(server, p.Servers.ScheduledTask)
}
if p.Servers.ScanSource != nil {
pluginv1.RegisterScanSourceServer(server, p.Servers.ScanSource)
}
```
- [ ] **Step 6: Run test to verify it passes**
Run: `go test ./pkg/pluginsdk/runtime/ -run TestGRPCServerRegistersScanSource -v`
Expected: PASS.
- [ ] **Step 7: Commit**
```bash
git add pkg/pluginsdk/runtime/runtime.go pkg/pluginsdk/runtime/runtime_test.go
git commit -m "feat(runtime): serve and dial scan_source.v1 capability"
```
---
## Task 4: Prove a `scan_source.v1` manifest loads
Guards the end-to-end path: a plugin manifest declaring the new capability must pass the loader (it would not before Task 2's allowlist entry).
**Files:**
- Modify: `pkg/pluginsdk/manifest/manifest_test.go`
- [ ] **Step 1: Write the failing test**
The test file is package `manifest_test` (external) and loads from a JSON literal via `manifest.Load([]byte)`. Append this test, mirroring `TestLoadAcceptsRequestRouterCapability` exactly with the type swapped to `scan_source.v1`:
```go
func TestLoadAcceptsScanSourceCapability(t *testing.T) {
raw := []byte(`{
"plugin_id": "silo.example",
"version": "1.0.0",
"silo_api_version": "v1",
"capabilities": [
{"type": "scan_source.v1", "id": "arr", "display_name": "X", "description": "Y"}
]
}`)
m, err := manifest.Load(raw)
if err != nil {
t.Fatalf("Load returned unexpected error: %v", err)
}
if len(m.GetCapabilities()) != 1 {
t.Fatalf("expected 1 capability, got %d", len(m.GetCapabilities()))
}
if got := m.GetCapabilities()[0].GetType(); got != "scan_source.v1" {
t.Fatalf("capability type = %q, want scan_source.v1", got)
}
}
```
- [ ] **Step 2: Run test to verify it fails (before this task's dependency was in place) / passes now**
Run: `go test ./pkg/pluginsdk/manifest/ -run TestLoadAcceptsScanSourceCapability -v`
Expected: PASS (Task 2 already added `scan_source.v1` to `KnownTypes`). If it FAILS with an unknown-capability-type error, Task 2 was not completed correctly — fix that first.
- [ ] **Step 3: Commit**
```bash
git add pkg/pluginsdk/manifest/manifest_test.go
git commit -m "test(manifest): accept scan_source.v1 capability in manifests"
```
---
## Task 5: Full build + test sweep
**Files:** none (verification only)
- [ ] **Step 1: Build everything**
Run: `go build ./...`
Expected: exits 0.
- [ ] **Step 2: Run the full SDK test suite**
Run: `go test ./...`
Expected: all packages `ok`. Confirms the additive change broke no existing capability, manifest, or runtime test.
- [ ] **Step 3: Vet + format**
Run: `go vet ./... && gofmt -l pkg proto 2>/dev/null; test -z "$(gofmt -l pkg)"`
Expected: vet clean; `gofmt -l pkg` prints nothing.
---
## Task 6: Tag the release
Downstream repos (silo-server host, arr plugin) consume the SDK as a tagged module version.
**Files:** none (release action)
- [ ] **Step 1: Confirm the version to tag**
Check the latest tag: `git tag --list 'v*' | sort -V | tail -3`. The current consumed version is `v0.4.0`; this additive feature warrants a minor bump to **`v0.5.0`** (no breaking changes).
- [ ] **Step 2: Tag and push**
```bash
git tag v0.5.0
git push origin v0.5.0
```
- [ ] **Step 3: Record the contract for downstream plans**
Note in the host and arr-plugin plans that they must require `github.com/Silo-Server/silo-plugin-sdk v0.5.0` and use:
- `capability.ScanSource` (`"scan_source.v1"`)
- `runtime.CapabilityServers{ ScanSource: <impl> }` (plugin side)
- `client.ScanSource()` → `pluginv1.ScanSourceClient.PollChanges(ctx, &pluginv1.PollChangesRequest{CapabilityId, Marker})` (host side)
- response fields `PollChangesResponse.ChangedPaths []string`, `PollChangesResponse.NextMarker string`
---
## Self-Review notes
- **Spec coverage:** Implements spec §5 (the `scan_source.v1` contract: `PollChanges`, opaque marker, Silo-native paths) and §11 (additive-only; registered in `KnownTypes`; no existing capability modified). Spec §12 build-order step 1 (tag the SDK) is Task 6.
- **Out of scope here:** the host engine (spec §7), connections (§8), the Autoscan category (§9), and the arr plugin (§10) are separate per-repo plans built against the `v0.5.0` tag produced by Task 6.
- **Marker is opaque end-to-end:** the proto carries `marker`/`next_marker` as plain strings with no server-side interpretation, satisfying the "any provider's bookmark" requirement.
docs/superpowers/plans/2026-06-03-autoscan-arr-plugin.md
+265
View File
@@ -0,0 +1,265 @@
# Sonarr/Radarr Scan-Source Plugin Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Build a new installable Silo plugin that implements `scan_source.v1` for Sonarr/Radarr: when the host pulls, it polls arr `/history`, extracts imported + renamed paths, applies its own path rewrites, and returns Silo-native paths + a marker.
**Architecture:** A standalone Go module structured like `silo-plugin-tmdb` — a `manifest.json` declaring `scan_source.v1`, a `main.go` serving `runtime.CapabilityServers{Runtime, ScanSource}` via `runtime.Serve`, and the arr-specific logic ported from the closed silo-server PR #43 (`internal/autoscan/history.go`, `rewrite.go`, `suggest.go`).
**Tech Stack:** Go, `github.com/Silo-Server/silo-plugin-sdk` (`v0.5.0`, the `scan_source.v1` capability), `hashicorp/go-plugin` runtime (via the SDK), arr HTTP API (`X-Api-Key`).
Commands assume the **new plugin repository root** is the cwd (the `silo-plugin-autoscan-arr` checkout, a sibling of this repo like the `silo-plugin-sdk` checkout).
---
## Prerequisites
1. **SDK `v0.5.0`.** Needs `scan_source.v1` from `silo-plugin-sdk` PR #2. Until tagged, depend on the local checkout via replace:
```bash
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk
```
This points at the `silo-plugin-sdk` checkout (a sibling of this repo). Task 8 finalizes to `v0.5.0`.
2. **Template:** the `silo-plugin-tmdb` checkout (a sibling of this repo) — copy its `main.go` runtime scaffold, `Makefile`, and `manifest.json` shape.
3. **Salvage source:** the arr logic is on silo-server `main` at `internal/autoscan/history.go` (imports + renames extraction, bounded window), `rewrite.go` (`applyRewrites`, `normalizeSeparators`), `suggest.go`/`suggest_deps.go` (root-folder suffix-match suggester). Port these verbatim where possible.
4. **New repo:** create `Silo-Server/silo-plugin-autoscan-arr` (or fork pattern). The plugin is registered in the `silo-plugins` catalog in a separate follow-up (Task 9 notes it).
---
## File Structure
| File | Responsibility | Action |
|---|---|---|
| `go.mod` | module `github.com/Silo-Server/silo-plugin-autoscan-arr`; SDK dep | Create |
| `manifest.json` | declares `scan_source.v1` capability + `global_config_schema` (path rewrites only; connection arrives per request) | Create |
| `main.go` | `runtimeServer` (GetManifest/Configure) + `scanSourceServer` (PollChanges); `runtime.Serve` | Create |
| `internal/arr/history.go` | poll `/history`, imports + renames, bounded window | Create (port) |
| `internal/arr/rewrite.go` | `applyRewrites`, `normalizeSeparators` | Create (port) |
| `internal/arr/client.go` | arr HTTP client (`X-Api-Key`, response cap, timeout) | Create |
| `internal/config/config.go` | parse Configure payload → `{rewrites}` (connection comes per request, not config) | Create |
| `Makefile` | build (cross-platform), like tmdb | Create |
---
## Task 1: Scaffold the plugin (serves an empty `scan_source.v1`)
**Files:** Create `go.mod`, `manifest.json`, `main.go`, `Makefile`.
- [ ] **Step 1: Module + SDK dep**
```bash
go mod init github.com/Silo-Server/silo-plugin-autoscan-arr
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk # the silo-plugin-sdk checkout, a sibling of this repo
go get github.com/Silo-Server/silo-plugin-sdk@v0.4.0 # replace points at the scan_source branch checkout
```
- [ ] **Step 2: manifest.json**
Mirror tmdb's shape, declaring the scan source:
```json
{
"plugin_id": "silo.autoscan.arr",
"version": "0.1.0",
"checksum": "__CHECKSUM__",
"silo_api_version": "v1",
"supported_platforms": [
{"os": "linux", "arch": "amd64"},
{"os": "linux", "arch": "arm64"},
{"os": "darwin", "arch": "arm64"}
],
"capabilities": [
{
"type": "scan_source.v1",
"id": "arr",
"display_name": "Sonarr / Radarr",
"description": "Triggers Silo rescans from Sonarr/Radarr import and rename history."
}
],
"global_config_schema": [
{"key": "path_rewrites", "type": "json", "required": false, "display_name": "Path rewrites"}
]
}
```
> **Connection is NOT plugin config.** The arr base URL + API key are delivered by the host in each `PollChangesRequest.connection` (resolved from the operator's Autoscan connection — own creds or a reused Requests link). The plugin reads them from the request and stores no credentials. Only path rewrites (and other provider-specific behaviour) live in plugin config.
(Confirm the exact `global_config_schema` entry shape against `silo-plugin-sdk` `pkg/pluginsdk/config` — adapt field names to the SDK's config schema type.)
- [ ] **Step 3: main.go scaffold**
Copy tmdb's `runtimeServer` (GetManifest from embedded manifest; Configure stores config) and add a `scanSourceServer` returning empty for now:
```go
type scanSourceServer struct{ cfg *config.Config }
func (s *scanSourceServer) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
return &pluginv1.PollChangesResponse{}, nil
}
func main() {
manifest := mustLoadManifest()
rt := &runtimeServer{manifest: manifest}
runtime.Serve(runtime.ServeConfig{
Servers: runtime.CapabilityServers{
Runtime: rt,
ScanSource: &scanSourceServer{cfg: rt.cfg},
},
})
}
```
- [ ] **Step 4: Build**
Run: `go build ./...`
Expected: exits 0.
- [ ] **Step 5: Commit**
```bash
git init && git add -A
git commit -m "feat: scaffold autoscan-arr scan_source.v1 plugin"
```
---
## Task 2: Config parsing (Configure → path rewrites only)
**Files:** Create `internal/config/config.go`, `internal/config/config_test.go`.
> Connection (base_url/api_key) is NOT config — it arrives per poll in `PollChangesRequest.connection` (Task 5). Plugin config carries only path rewrites.
- [ ] **Step 1: Failing test**
```go
func TestParseConfig(t *testing.T) {
cfg, err := Parse(map[string]string{
"path_rewrites": `[{"from":"/mnt/arr/tv","to":"/mnt/media/tv"}]`,
})
if err != nil { t.Fatalf("Parse: %v", err) }
if len(cfg.Rewrites) != 1 || cfg.Rewrites[0].From != "/mnt/arr/tv" { t.Fatalf("bad rewrites: %+v", cfg.Rewrites) }
}
```
- [ ] **Step 2: Run → fail.** `go test ./internal/config/` → undefined `Parse`.
- [ ] **Step 3: Implement** `Parse(map[string]string) (*Config, error)` with `Config{Rewrites []Rewrite}` where `Rewrite{From, To string}`; unmarshal `path_rewrites` JSON. Wire `runtimeServer.Configure` to store the parsed config so `scanSourceServer` reads the rewrites.
- [ ] **Step 4: Run → pass.**
- [ ] **Step 5: Commit** `feat(config): parse arr connection and path rewrites`.
---
## Task 3: arr history client (imports + renames, bounded window)
**Files:** Create `internal/arr/client.go`, `internal/arr/history.go`, `internal/arr/history_test.go`.
Port from silo-server `main:internal/autoscan/history.go` (the `ChangedPaths` logic: `downloadFolderImported.importedPath`; `episodeFileRenamed`/`movieFileRenamed` → `path` + `sourcePath`; deletes ignored) and the client (`X-Api-Key`, 1 MiB cap, timeout). The marker is an RFC3339 timestamp; empty marker ⇒ "now".
- [ ] **Step 1: Failing test** — copy the salvaged `history_test.go` (`TestArrHistoryChangedPaths`: imports + both rename paths returned, `grabbed`/`episodeFileDeleted` ignored), renamed to this package, asserting the returned `(paths, nextMarker)`.
- [ ] **Step 2: Run → fail.**
- [ ] **Step 3: Implement** `ChangedPaths(ctx, baseURL, apiKey, since time.Time) (paths []string, newest time.Time, err error)` — port `history.go`, additionally returning the newest history timestamp as the next marker. Apply the bounded window (24h max-lookback floor + overlap buffer) here.
- [ ] **Step 4: Run → pass.**
- [ ] **Step 5: Commit** `feat(arr): poll history for imports and renames`.
---
## Task 4: Path rewrites (→ Silo-native paths)
**Files:** Create `internal/arr/rewrite.go`, `internal/arr/rewrite_test.go`.
Port `applyRewrites` (boundary-safe prefix: `path==trimmed || HasPrefix(path, trimmed+"/")`) and `normalizeSeparators` from silo-server `main:internal/autoscan/rewrite.go`.
- [ ] **Step 1: Failing test** — copy salvaged `rewrite_test.go` cases.
- [ ] **Step 2: Run → fail.**
- [ ] **Step 3: Implement** the port.
- [ ] **Step 4: Run → pass.**
- [ ] **Step 5: Commit** `feat(arr): boundary-safe path rewrites`.
---
## Task 5: Wire `PollChanges` end-to-end
**Files:** Modify `main.go` (`scanSourceServer.PollChanges`), add `main_test.go`.
- [ ] **Step 1: Failing test** — a `scanSourceServer` (rewrites configured) called with a `PollChangesRequest` whose `Connection` points at a stub arr server (httptest); assert `PollChanges` with empty marker returns rewritten Silo-native paths and a non-empty `next_marker`, and that the request actually hit the stub at `connection.base_url` with `X-Api-Key: connection.api_key`. A second call with the returned marker queries arr with the right `since`. Also assert a request with a nil/empty `Connection` returns an error (no credentials to poll with).
- [ ] **Step 2: Run → fail.**
- [ ] **Step 3: Implement** — read the connection from the request, not from config:
```go
func (s *scanSourceServer) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
conn := req.GetConnection()
if conn == nil || conn.GetBaseUrl() == "" {
return nil, fmt.Errorf("scan_source: no connection supplied")
}
since := time.Time{} // empty marker => now (history client floors to now)
if m := req.GetMarker(); m != "" {
if t, err := time.Parse(time.RFC3339, m); err == nil { since = t }
}
raw, newest, err := arr.ChangedPaths(ctx, conn.GetBaseUrl(), conn.GetApiKey(), since)
if err != nil { return nil, err }
out := make([]string, 0, len(raw))
for _, p := range raw {
out = append(out, arr.ApplyRewrites(arr.NormalizeSeparators(p), s.cfg.Rewrites))
}
return &pluginv1.PollChangesResponse{ChangedPaths: out, NextMarker: newest.UTC().Format(time.RFC3339)}, nil
}
```
- [ ] **Step 4: Run → pass.**
- [ ] **Step 5: Commit** `feat: PollChanges returns Silo-native paths from arr history`.
---
## Task 6: Rewrite suggester ("Sync from arr") — root folders
**Files:** Create `internal/arr/suggest.go`, `suggest_test.go`.
Port `suggestRewrites` (suffix-match arr root folders → Silo folders) from `main:internal/autoscan/suggest.go`. **Open question (spec §14 risk #2):** how the operator triggers "Sync" and how the plugin reaches Silo's media-folder list.
- [ ] **Step 1:** Decide the delivery mechanism — recommended: the plugin reads Silo's folder list via the host library-listing service (`pluginhost` library lister, reachable through the SDK `runtime.Host()` client). Confirm that path exists in `v0.5.0`; if not, scope "Sync" to a follow-up and ship Tasks 1–5 (manual rewrites only) first.
- [ ] **Step 2:** Port the suffix-match suggester + its tests (unique/ambiguous/covered/no-op/normalization cases from `suggest_test.go`).
- [ ] **Step 3:** Expose it — either via an `http_routes.v1` capability the admin UI calls, or as part of the plugin config flow. Document the chosen mechanism.
- [ ] **Step 4:** Tests pass; commit `feat(arr): suffix-match rewrite suggester`.
> If §14 risk #2 is unresolved at execution time, **skip this task**, ship manual rewrites (Tasks 1–5), and file the suggester as a follow-up. The plugin is fully functional without it.
---
## Task 7: Cross-repo smoke test against the host
**Files:** none (verification).
- [ ] **Step 1:** Build the plugin binary; install it into a dev silo-server (with the Part-1 backend) via the plugin installer.
- [ ] **Step 2:** Configure a source pointing at a real/stub arr; trigger an autoscan poll; confirm the host enqueues a scan for an imported path. (This exercises the full SDK contract end-to-end.)
- [ ] **Step 3:** Note results; no commit unless fixes are needed.
---
## Task 8: Finalize SDK dependency
- [ ] After `silo-plugin-sdk v0.5.0` is tagged: `go mod edit -dropreplace ...; go get .../silo-plugin-sdk@v0.5.0; go mod tidy; go build ./...`.
- [ ] Commit `build: depend on silo-plugin-sdk v0.5.0`.
- [ ] Do not publish/tag the plugin with a `replace` directive in `go.mod`.
---
## Task 9: Catalog registration (follow-up, separate repo)
- [ ] Add `silo.autoscan.arr` to the `silo-plugins` catalog manifest so it is installable. This is a separate PR in the `silo-plugins` repo (not this module) — note it; out of scope for this plan's repo.
---
## Self-Review notes
- **Spec coverage:** §10 (arr plugin: history imports + renames, bounded window, rewrites, Silo-native paths) Tasks 3–5; §5 contract (`PollChanges`, opaque marker = RFC3339 timestamp, first-run "now") Task 5; "Sync from arr" §10/§14-risk-2 Task 6 (with an explicit skip path).
- **Salvage:** history/rewrite/suggest ported from the closed PR (Tasks 3,4,6) — exactly the files the host backend plan deletes (Part-1 Task 8).
- **Risk:** §14 risk #1 (one plugin installation per arr server vs. many) — this plugin holds one connection per installation/config; multiple arr servers ⇒ multiple installs unless the runtime supports multiple capability instances. §14 risk #2 (Sync UI) gated in Task 6 with a clean skip.
- **Type consistency:** `Config{Rewrites}` (no creds — connection arrives in `PollChangesRequest.connection`), `Rewrite{From,To}`, `arr.ChangedPaths(ctx, baseURL, apiKey, since) ([]string, time.Time, error)`, `arr.ApplyRewrites`/`NormalizeSeparators`, `scanSourceServer.PollChanges` consistent across Tasks 2–6.
docs/superpowers/plans/2026-06-03-autoscan-host-ui.md
+209
View File
@@ -0,0 +1,209 @@
# Autoscan Admin UI Implementation Plan (Part 2 of 2)
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Give Autoscan its own admin category (page + sidebar nav), separate from Requests: manage Connections (reuse a Requests arr server or enter own), Sources (per scan-source plugin: enable, interval, connection), and global settings — backed by the Part-1 v2 API.
**Architecture:** Extract the Autoscan tab out of `AdminRequests.tsx` into a standalone `AdminAutoscan.tsx` page; register a route and a sidebar item; rebuild `useAutoscan.ts` hooks + `types.ts` against the v2 endpoints (`/admin/autoscan/{settings,connections,sources,trigger,status}`).
**Tech Stack:** React + TypeScript, React Query (`@tanstack/react-query`), the project's `ui/` component library (`Tabs`, `Table`, `Card`, etc.), `react-router` routes in `App.tsx`, 2-space indent / double quotes / 100-col (`web/.prettierrc`).
Commands assume the silo-server repository root is the cwd. **Depends on Part 1 (backend)** being implemented (the v2 endpoints must exist).
---
## Prerequisites
1. **Part-1 backend merged/available** — the v2 endpoints and response shapes (connections with `request_integration_id`, sources with `marker`/`last_error`, settings with `default_poll_interval_seconds`).
2. **Reference the existing autoscan tab** at `web/src/pages/AdminRequests.tsx` (the `autoscan` `TabsContent`, the `AutoscanTab`/`AutoscanSourceEditor` components) and hooks at `web/src/hooks/queries/useAutoscan.ts` — these are the starting material; they are reshaped, not discarded.
3. **Run before MR:** `cd web && pnpm run lint && pnpm run format:check`. Note the 20 pre-existing failing frontend tests are unrelated (project memory); do not be alarmed if they appear — only newly-introduced failures matter.
---
## File Structure
| File | Responsibility | Action |
|---|---|---|
| `web/src/api/types.ts` | `AutoscanConnection`, `AutoscanSource`, `AutoscanSettings` (v2 shapes) | Modify |
| `web/src/hooks/queries/useAutoscan.ts` | hooks for settings/connections/sources/trigger/status | Rewrite |
| `web/src/hooks/queries/keys.ts` | query keys for connections/sources | Modify |
| `web/src/pages/AdminAutoscan.tsx` | the new standalone Autoscan page (Connections + Sources + Settings tabs) | Create |
| `web/src/pages/admin/autoscan/ConnectionsPanel.tsx` | list/add/edit/delete connections (own or Requests-linked) | Create |
| `web/src/pages/admin/autoscan/SourcesPanel.tsx` | per-source enable, interval, connection binding, status | Create |
| `web/src/pages/AdminRequests.tsx` | remove the Autoscan tab | Modify |
| `web/src/App.tsx` | add `<Route path="autoscan" element={<AdminAutoscan />} />` | Modify |
| `web/src/components/AdminSidebar.tsx` | add the Autoscan nav item under "Content" | Modify |
---
## Task 1: v2 types + hooks
**Files:** Modify `web/src/api/types.ts`, `web/src/hooks/queries/keys.ts`; rewrite `web/src/hooks/queries/useAutoscan.ts`.
- [ ] **Step 1: Types**
Add to `web/src/api/types.ts`:
```ts
export interface AutoscanSettings {
enabled: boolean;
default_poll_interval_seconds: number;
debounce_seconds: number;
}
export interface AutoscanConnection {
id: string;
name: string;
kind: string;
// present when the connection has its own credentials:
base_url?: string;
// present when linked to a Requests integration (live reuse):
request_integration_id?: string | null;
// NOTE: api_key_ref / resolved keys are never sent by the backend.
}
export interface AutoscanSource {
id: string;
installation_id: number;
capability_id: string;
connection_id: string;
enabled: boolean;
poll_interval_seconds: number | null;
last_run_at: string | null;
last_error: string | null;
}
```
- [ ] **Step 2: Hooks**
Rewrite `useAutoscan.ts` with: `useAutoscanSettings`/`useUpdateAutoscanSettings`, `useAutoscanConnections`/`useCreateAutoscanConnection`/`useUpdateAutoscanConnection`/`useDeleteAutoscanConnection`, `useAutoscanSources`/`useUpdateAutoscanSource`, `useAutoscanStatus`, `useTriggerAutoscan`. Follow the existing hook style (the current `useAutoscan.ts` is the template for query/mutation + invalidation). Add query keys in `keys.ts`.
- [ ] **Step 3: Typecheck**
Run: `cd web && pnpm exec tsc --noEmit`
Expected: no type errors in the autoscan files.
- [ ] **Step 4: Commit**
```bash
git add web/src/api/types.ts web/src/hooks/queries/useAutoscan.ts web/src/hooks/queries/keys.ts
git commit -m "feat(web): autoscan v2 types and query hooks"
```
---
## Task 2: Connections panel
**Files:** Create `web/src/pages/admin/autoscan/ConnectionsPanel.tsx`.
- [ ] **Step 1: Build the panel**
A `Card` + `Table` listing connections (name, kind, and a "Reused from Requests" / "Own" badge derived from `request_integration_id`). An "Add connection" dialog offers two modes:
- **Reuse from Requests** — a select populated from the existing request-integrations hook (filter to arr kinds), storing `request_integration_id`.
- **Enter own** — name + URL + API key fields, posting own credentials.
Edit + delete actions per row. Never render any key material (the backend doesn't send it).
- [ ] **Step 2: Lint/format/typecheck**
Run: `cd web && pnpm run lint && pnpm exec tsc --noEmit`
Expected: clean.
- [ ] **Step 3: Commit** `feat(web): autoscan connections panel (reuse or own)`.
---
## Task 3: Sources panel
**Files:** Create `web/src/pages/admin/autoscan/SourcesPanel.tsx`.
- [ ] **Step 1: Build the panel**
A `Table` of sources, one row per installed `scan_source` capability instance: plugin/capability label, a connection `<select>` (bound to `connection_id`), an enable toggle, a poll-interval input (blank = settings default), and a status column (`last_run_at`, `last_error`). Reuse the spinner/"Syncing…" affordances from the old `AutoscanSourceEditor` where relevant. Wire to `useAutoscanSources`/`useUpdateAutoscanSource`.
- [ ] **Step 2: Lint/format/typecheck** — clean.
- [ ] **Step 3: Commit** `feat(web): autoscan sources panel`.
---
## Task 4: The Autoscan page + global settings + manual trigger
**Files:** Create `web/src/pages/AdminAutoscan.tsx`.
- [ ] **Step 1: Compose the page**
A page with a `Tabs` (or sections): **Sources**, **Connections**, **Settings**. Settings = global enable toggle, default poll interval, debounce. A "Run now" button calling `useTriggerAutoscan` (202 → toast "Autoscan triggered"). Mirror the header/layout of `AdminRequests.tsx` for visual consistency.
- [ ] **Step 2: Lint/format/typecheck** — clean.
- [ ] **Step 3: Commit** `feat(web): standalone Autoscan admin page`.
---
## Task 5: Route + sidebar nav
**Files:** Modify `web/src/App.tsx`, `web/src/components/AdminSidebar.tsx`.
- [ ] **Step 1: Route**
In `web/src/App.tsx`, next to `<Route path="requests" element={<AdminRequests />} />` (l.374), add:
```tsx
<Route path="autoscan" element={<AdminAutoscan />} />
```
with the matching `import AdminAutoscan from "@/pages/AdminAutoscan";`.
- [ ] **Step 2: Sidebar item**
In `web/src/components/AdminSidebar.tsx`, under the **Content** group (after the Requests item, ~l.112), add:
```tsx
{
label: "Autoscan",
icon: <RefreshCw className="h-[18px] w-[18px]" />,
href: "/admin/autoscan",
},
```
(Import an appropriate icon from the icon set already used in the file.)
- [ ] **Step 3: Verify navigation**
Run: `cd web && pnpm run build`
Expected: build succeeds. Manually confirm `/admin/autoscan` renders and the sidebar item highlights.
- [ ] **Step 4: Commit** `feat(web): route and sidebar nav for Autoscan category`.
---
## Task 6: Remove the Autoscan tab from Requests
**Files:** Modify `web/src/pages/AdminRequests.tsx`.
- [ ] **Step 1: Excise the tab**
Remove the `<TabsTrigger value="autoscan">` (l.146) and its `<TabsContent value="autoscan">`, the `AutoscanTab`/`AutoscanSourceEditor` definitions (now superseded by the new panels), and `autoscan` from `ADMIN_REQUEST_TABS`. Remove now-unused imports.
- [ ] **Step 2: Lint/format/typecheck/build**
Run: `cd web && pnpm run lint && pnpm run format:check && pnpm run build`
Expected: clean; build succeeds; no dangling references to removed components.
- [ ] **Step 3: Commit** `refactor(web): move Autoscan out of Requests into its own category`.
---
## Task 7: Final sweep
- [ ] **Step 1:** `cd web && pnpm run lint && pnpm run format:check && pnpm run build` — all clean.
- [ ] **Step 2:** `cd web && pnpm test` — confirm no **newly** failing tests (the 20 pre-existing failures, per project memory, are unrelated; compare against baseline).
- [ ] **Step 3:** Manual pass: add a connection (both modes), bind a source, toggle enable, "Run now", confirm status updates.
---
## Self-Review notes
- **Spec coverage:** §9 (Autoscan as its own category, decoupled from Requests; Connections reuse-or-own; per-source enable/interval/status) — Tasks 1–6. The reuse-from-Requests UI (Task 2) surfaces the soft link from spec §8.
- **Salvage:** the old `AutoscanTab`/`AutoscanSourceEditor` and `useAutoscan.ts` are reshaped (Tasks 1–3), then the Requests tab is removed (Task 6).
- **Out of scope:** plugin-side config UI (the path-rewrite table lives on the plugin's own settings screen per spec §10 — that is the arr-plugin plan / spec §14 risk #2, not this page).
- **Type consistency:** `AutoscanConnection{request_integration_id?}`, `AutoscanSource{connection_id,poll_interval_seconds,last_error}`, `AutoscanSettings{default_poll_interval_seconds}` used consistently across hooks (Task 1) and panels (Tasks 2–4).
docs/superpowers/plans/2026-06-05-autoscan-source-labels.md
+1031
View File
File diff suppressed because it is too large Load Diff
docs/superpowers/plans/2026-06-06-audiobooks-stacked-pr-split.md
+174
View File
@@ -0,0 +1,174 @@
# Audiobooks Stacked PR Split Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Replace the oversized audiobook PR with a stacked series of smaller, reviewable PRs.
**Architecture:** Preserve the existing `feat/audiobooks` work as the source branch, then create branch cut points that expose one logical layer at a time. Each PR targets the previous branch so reviewers see only the incremental diff.
**Tech Stack:** Git, GitHub CLI, Go, pnpm/Vite, PostgreSQL migrations.
---
### Task 1: Close The Oversized PR
**Files:**
- Modify: GitHub PR #17 only.
- [ ] **Step 1: Post a replacement note**
Run:
```bash
gh pr comment 17 --repo Silo-Server/silo-server --body 'Closing this oversized PR in favor of a stacked series of smaller PRs. The existing branch is preserved as the source-of-truth while the stack is rebuilt into logical review units.'
```
Expected: GitHub prints the created comment URL.
- [ ] **Step 2: Close PR #17 without deleting the branch**
Run:
```bash
gh pr close 17 --repo Silo-Server/silo-server
```
Expected: PR #17 state becomes `CLOSED`; branch `feat/audiobooks` remains available.
### Task 2: Create Stacked Branches
**Files:**
- Modify: remote Git branches only.
- [ ] **Step 1: Create branch 1, foundation**
Branch name: `stack/audiobooks-foundation`
Scope:
- audiobook feature flag/settings
- audiobook and podcast schema foundations
- scanner support
- audiobook media item write path
- no native UI
- no ABS listener
- [ ] **Step 2: Create branch 2, native UI MVP**
Branch name: `stack/audiobooks-native-ui`
Base: `stack/audiobooks-foundation`
Scope:
- `/api/v1/audiobooks`
- audiobook detail/progress endpoints
- audiobook frontend route/sidebar/player MVP
- [ ] **Step 3: Create branch 3, ABS core**
Branch name: `stack/audiobooks-abs-core`
Base: `stack/audiobooks-native-ui`
Scope:
- ABS auth/login/refresh/logout
- ABS listener and feature-flag listener gating
- ABS library/items/play/progress
- ABS access control and security hardening
- [ ] **Step 4: Create branch 4, ABS collections**
Branch name: `stack/audiobooks-abs-collections`
Base: `stack/audiobooks-abs-core`
Scope:
- ABS bookmarks
- ABS collections
- ABS playlists
- ABS smart collections
- unified collection migration
- [ ] **Step 5: Create branch 5, extras and polish**
Branch name: `stack/audiobooks-extras`
Base: `stack/audiobooks-abs-collections`
Scope:
- podcasts/RSS
- stats
- author/series extras
- catalog audiobook filters/typeahead
- performance and enrichment cleanup
### Task 3: Open The Stacked PRs
**Files:**
- Modify: GitHub PRs only.
- [ ] **Step 1: Open PR 1**
Run:
```bash
gh pr create --repo Silo-Server/silo-server --base main --head stack/audiobooks-foundation --title 'feat(audiobooks): foundation and scanner support' --body 'First PR in the audiobook stack. Adds the schema/settings/scanner foundation without native UI or ABS compatibility.'
```
- [ ] **Step 2: Open PR 2**
Run:
```bash
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-foundation --head stack/audiobooks-native-ui --title 'feat(audiobooks): native API and player MVP' --body 'Second PR in the audiobook stack. Adds native Silo audiobook endpoints and the MVP web player UI.'
```
- [ ] **Step 3: Open PR 3**
Run:
```bash
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-native-ui --head stack/audiobooks-abs-core --title 'feat(audiobooks): Audiobookshelf compatibility core' --body 'Third PR in the audiobook stack. Adds ABS auth, listener, library/items/play/progress, and security hardening.'
```
- [ ] **Step 4: Open PR 4**
Run:
```bash
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-abs-core --head stack/audiobooks-abs-collections --title 'feat(audiobooks): ABS collections, playlists, and smart collections' --body 'Fourth PR in the audiobook stack. Adds ABS bookmarks, collections, playlists, smart collections, and unified collection storage.'
```
- [ ] **Step 5: Open PR 5**
Run:
```bash
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-abs-collections --head stack/audiobooks-extras --title 'feat(audiobooks): podcasts, catalog filters, and enrichment polish' --body 'Final PR in the audiobook stack. Adds podcast/RSS extras, audiobook catalog filters/typeahead, and scan/enrichment performance polish.'
```
### Task 4: Verify And Report
**Files:**
- Modify: none.
- [ ] **Step 1: Verify PR states**
Run:
```bash
gh pr list --repo Silo-Server/silo-server --state open --search 'audiobooks in:title' --json number,title,headRefName,baseRefName,url
```
Expected: five open stacked PRs with the base/head chain shown above.
- [ ] **Step 2: Run branch verification where feasible**
Run:
```bash
go test ./...
cd web && pnpm run build
```
Expected: both commands pass on the final stack tip.
docs/superpowers/plans/2026-06-06-marker-introdb-readpath-correctness.md
+617
View File
@@ -0,0 +1,617 @@
# TheIntroDB read-path correctness (Phase 1 / Layer A) — implementation plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Fix the three read-path correctness gaps in the TheIntroDB provider so it matches the
real v3 API contract: (1) honor **TVDB** ids in lookups (today they are silently dropped),
(2) capture and use the **real per-segment `confidence`** instead of a hardcoded `0.9`, and
(3) when the API returns multiple candidate segments, **pick the most-submitted / highest-confidence**
one instead of the first usable. This is Phase 1 of
[the marker sources & contribution design](../specs/2026-06-06-marker-sources-and-contribution-design.md);
it stands alone, ships value immediately (anime / TheTVDB-first libraries start getting markers),
and unblocks the "query-all / best-wins" merge in later phases.
**Architecture:** All changes are contained within `internal/markers/introdb` plus one added
field on `markers.Marker`. The resolver already populates the TVDB id into
`Request.ExternalIDs` — only the provider and HTTP client drop it. The HTTP `Client` gains a
`tvdbID` argument on `FetchEpisode`/`FetchMovie` (query preference `tmdb → tvdb → imdb`, cache
keys extended); the provider reads `ExternalIDKeyTVDB`; `segmentTimestamps` gains `confidence`
and `submission_count`; `pickMarker` uses real confidence (with a named default) and ranks
candidates. The write path (`BuildUpdatePayload` / `UpsertMarkers`) is **not** touched here.
**Out of scope (deferred to Phase 2):** threading *per-segment* confidence through
`MarkerUpdatePayload` and `UpsertMarkers`. That refactor only becomes observable once
`Registry.FetchMerged` can combine segments from different providers; with a single provider
today the existing "max confidence across segments" collapse is equivalent, and changing it now
would break `TestBuildUpdatePayloadAggregatesConfidence` for no functional gain. It lands with
the merge dispatch in Phase 2.
**Tech Stack:** Go (the `internal/markers/introdb` package is pure Go — no libvips/CGO),
PostgreSQL is not involved (no migration), no frontend changes. Tests run in a throwaway Go
container (the host has no Go toolchain); a named volume caches module downloads.
Commands assume the repository root is the cwd.
---
## Running tests (host has no Go toolchain)
`internal/markers` and `internal/markers/introdb` are pure Go, so no libvips is needed:
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/... -v
```
To run a single test, append `-run TestName` and narrow the package path, e.g.
`go test ./internal/markers/introdb/ -run TestFetchEpisodeSendsTVDBWhenNoTMDB -v`.
---
## File structure
- `internal/markers/introdb/types.go` — **modify**. Add `Confidence` + `SubmissionCount` to `segmentTimestamps`; add the `defaultConfidence` const.
- `internal/markers/types.go` — **modify**. Add `SubmissionCount` to `markers.Marker`.
- `internal/markers/introdb/client.go` — **modify**. `tvdbID` arg on `FetchEpisode`/`FetchMovie`; query preference `tmdb → tvdb → imdb`; cache keys include tvdb.
- `internal/markers/introdb/provider.go` — **modify**. Read `ExternalIDKeyTVDB` and pass it through; rewrite `pickMarker` for real confidence + best-candidate.
- `internal/markers/introdb/client_test.go` — **create**. httptest-based client tests (TVDB query, TMDB preference, caching).
- `internal/markers/introdb/provider_test.go` — **create**. End-to-end provider tests (TVDB-only resolves, real/default confidence, best-candidate).
No migration, no wiring change (`cmd/silo/main.go` constructs the provider but never calls the
client directly), no frontend change.
---
## Task 1: Type plumbing — capture confidence + submission_count
**Files:**
- Modify: `internal/markers/introdb/types.go`
- Modify: `internal/markers/types.go`
This task is pure struct plumbing (no behavior yet); later tasks' tests exercise it. Verify by
compilation.
- [ ] **Step 1: Add the response fields + default constant in `introdb/types.go`**
Replace the `Algorithm` const block to add `defaultConfidence` after it:
```go
// Algorithm is the algorithm tag written alongside markers. The version
// suffix lets us invalidate or refresh markers if the upstream contract
// changes.
const Algorithm = "introdb:v3"
// defaultConfidence is applied when TheIntroDB omits a per-segment confidence
// in the /media response. Real per-segment confidence is preferred when present.
const defaultConfidence = 0.9
```
Replace the `segmentTimestamps` struct with the version that decodes the two extra fields the
v3 `/media` response carries:
```go
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
// Either bound may be nil — for intro/recap, start may be omitted (segment
// begins at file start); for credits/preview, end may be omitted (segment
// runs to file end). Confidence and SubmissionCount are optional per-segment
// quality signals used to pick among multiple candidates.
type segmentTimestamps struct {
StartMs *int64 `json:"start_ms,omitempty"`
EndMs *int64 `json:"end_ms,omitempty"`
Confidence *float64 `json:"confidence,omitempty"`
SubmissionCount *int `json:"submission_count,omitempty"`
}
```
- [ ] **Step 2: Add `SubmissionCount` to `markers.Marker`**
In `internal/markers/types.go`, replace the `Marker` struct:
```go
type Marker struct {
Kind MarkerKind
Start time.Duration
End time.Duration
Confidence float64
SubmissionCount int
}
```
- [ ] **Step 3: Verify the packages still compile**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go build ./internal/markers/...
```
Expected: no output, exit 0. (Existing `write_test.go` / `types_test.go` use field-keyed
literals, so the added field does not break them.)
- [ ] **Step 4: Commit**
```bash
git add internal/markers/introdb/types.go internal/markers/types.go
git commit -m "feat(markers): decode introdb per-segment confidence + submission_count"
```
---
## Task 2: TVDB lookups (gap #1)
**Files:**
- Modify: `internal/markers/introdb/client.go`
- Modify: `internal/markers/introdb/provider.go`
- Create: `internal/markers/introdb/client_test.go`
- [ ] **Step 1: Write the failing client tests**
Create `internal/markers/introdb/client_test.go`:
```go
package introdb
import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"sync/atomic"
"testing"
)
func TestFetchEpisodeSendsTVDBWhenNoTMDB(t *testing.T) {
var gotQuery url.Values
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotQuery = r.URL.Query()
_, _ = w.Write([]byte(`{"type":"episode"}`))
}))
defer srv.Close()
c := NewClient("")
c.SetBaseURL(srv.URL)
if _, err := c.FetchEpisode(context.Background(), "", "55555", "tt1234567", 2, 3, 0); err != nil {
t.Fatalf("FetchEpisode: %v", err)
}
if gotQuery.Get("tvdb_id") != "55555" {
t.Errorf("tvdb_id = %q, want 55555", gotQuery.Get("tvdb_id"))
}
if gotQuery.Get("tmdb_id") != "" {
t.Errorf("tmdb_id = %q, want empty", gotQuery.Get("tmdb_id"))
}
if gotQuery.Get("imdb_id") != "" {
t.Errorf("imdb_id should be omitted when tvdb present, got %q", gotQuery.Get("imdb_id"))
}
}
func TestFetchEpisodePrefersTMDBOverTVDBAndIMDB(t *testing.T) {
var gotQuery url.Values
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotQuery = r.URL.Query()
_, _ = w.Write([]byte(`{"type":"episode"}`))
}))
defer srv.Close()
c := NewClient("")
c.SetBaseURL(srv.URL)
if _, err := c.FetchEpisode(context.Background(), "111", "222", "tt333", 1, 1, 0); err != nil {
t.Fatalf("FetchEpisode: %v", err)
}
if gotQuery.Get("tmdb_id") != "111" {
t.Errorf("tmdb_id = %q, want 111", gotQuery.Get("tmdb_id"))
}
if gotQuery.Get("tvdb_id") != "" || gotQuery.Get("imdb_id") != "" {
t.Errorf("only tmdb_id expected, got tvdb=%q imdb=%q", gotQuery.Get("tvdb_id"), gotQuery.Get("imdb_id"))
}
}
func TestFetchMovieSendsTVDB(t *testing.T) {
var gotQuery url.Values
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotQuery = r.URL.Query()
_, _ = w.Write([]byte(`{"type":"movie"}`))
}))
defer srv.Close()
c := NewClient("")
c.SetBaseURL(srv.URL)
if _, err := c.FetchMovie(context.Background(), "", "888", "", 0); err != nil {
t.Fatalf("FetchMovie: %v", err)
}
if gotQuery.Get("tvdb_id") != "888" {
t.Errorf("tvdb_id = %q, want 888", gotQuery.Get("tvdb_id"))
}
}
func TestFetchEpisodeCachesByID(t *testing.T) {
var hits int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&hits, 1)
_, _ = w.Write([]byte(`{"type":"episode"}`))
}))
defer srv.Close()
c := NewClient("")
c.SetBaseURL(srv.URL)
for i := 0; i < 3; i++ {
if _, err := c.FetchEpisode(context.Background(), "", "999", "", 1, 1, 0); err != nil {
t.Fatalf("FetchEpisode: %v", err)
}
}
if hits != 1 {
t.Fatalf("server hits = %d, want 1 (cached after first)", hits)
}
}
```
- [ ] **Step 2: Run the tests to verify they fail**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/introdb/ -run 'TestFetch' -v
```
Expected: FAIL — compile error (`FetchEpisode`/`FetchMovie` still take the old 6/4-arg
signatures without `tvdbID`).
- [ ] **Step 3: Add `tvdbID` to `FetchEpisode` in `client.go`**
Replace the whole `FetchEpisode` method:
```go
// FetchEpisode looks up segment timestamps for a TV episode.
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
if tmdbID == "" && tvdbID == "" && imdbID == "" {
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
}
if season <= 0 || episode <= 0 {
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
}
q := url.Values{}
switch {
case tmdbID != "":
q.Set("tmdb_id", tmdbID)
case tvdbID != "":
q.Set("tvdb_id", tvdbID)
default:
q.Set("imdb_id", imdbID)
}
q.Set("season", strconv.Itoa(season))
q.Set("episode", strconv.Itoa(episode))
if durationMS > 0 {
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
}
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, tvdbID, imdbID, season, episode, durationMS))
}
```
- [ ] **Step 4: Add `tvdbID` to `FetchMovie` in `client.go`**
Replace the whole `FetchMovie` method:
```go
// FetchMovie looks up segment timestamps for a movie.
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
func (c *Client) FetchMovie(ctx context.Context, tmdbID, tvdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
if tmdbID == "" && tvdbID == "" && imdbID == "" {
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
}
q := url.Values{}
switch {
case tmdbID != "":
q.Set("tmdb_id", tmdbID)
case tvdbID != "":
q.Set("tvdb_id", tvdbID)
default:
q.Set("imdb_id", imdbID)
}
if durationMS > 0 {
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
}
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, tvdbID, imdbID, durationMS))
}
```
- [ ] **Step 5: Extend the cache-key helpers in `client.go`**
Replace both `cacheKeyEpisode` and `cacheKeyMovie`:
```go
func cacheKeyEpisode(tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) string {
switch {
case tmdbID != "":
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
case tvdbID != "":
return fmt.Sprintf("tvdb:%s:s%de%d:d%d", tvdbID, season, episode, durationMS)
default:
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
}
}
func cacheKeyMovie(tmdbID, tvdbID, imdbID string, durationMS int64) string {
switch {
case tmdbID != "":
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
case tvdbID != "":
return fmt.Sprintf("tvdb:movie:%s:d%d", tvdbID, durationMS)
default:
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
}
}
```
- [ ] **Step 6: Read TVDB in the provider and pass it through (`provider.go`)**
Replace the id-extraction block in `FetchMarkers` (the `tmdbID`/`imdbID` lines and the empty
guard):
```go
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
tvdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTVDB])
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
if tmdbID == "" && tvdbID == "" && imdbID == "" {
return markers.Result{}, nil
}
```
Then update the two client calls in the `switch req.Kind` block to pass `tvdbID`:
```go
resp, err = p.client.FetchEpisode(ctx, tmdbID, tvdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
```
```go
resp, err = p.client.FetchMovie(ctx, tmdbID, tvdbID, imdbID, durationMS)
```
- [ ] **Step 7: Run the client tests to verify they pass**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/introdb/ -run 'TestFetch' -v
```
Expected: PASS (all four `TestFetch*` tests).
- [ ] **Step 8: Commit**
```bash
git add internal/markers/introdb/client.go internal/markers/introdb/provider.go internal/markers/introdb/client_test.go
git commit -m "fix(markers): honor TVDB ids in TheIntroDB lookups"
```
---
## Task 3: Real confidence + best-candidate selection (gaps #2, #3)
**Files:**
- Modify: `internal/markers/introdb/provider.go` (`pickMarker`)
- Create: `internal/markers/introdb/provider_test.go`
- [ ] **Step 1: Write the failing provider tests**
Create `internal/markers/introdb/provider_test.go`:
```go
package introdb
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/markers"
)
func newProvider(t *testing.T, body string) *Provider {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
c := NewClient("")
c.SetBaseURL(srv.URL)
return NewProvider(c)
}
func episodeReq(ids map[string]string) markers.Request {
return markers.Request{
Kind: markers.ItemKindEpisode,
ExternalIDs: ids,
SeasonNumber: 1,
EpisodeNumber: 1,
Duration: 30 * time.Minute,
}
}
func TestProviderResolvesTVDBOnly(t *testing.T) {
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
res, err := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTVDB: "777"}))
if err != nil {
t.Fatalf("FetchMarkers: %v", err)
}
if len(res.Markers) != 1 || res.Markers[0].Kind != markers.MarkerKindIntro {
t.Fatalf("expected one intro marker, got %+v", res.Markers)
}
}
func TestProviderUsesRealConfidence(t *testing.T) {
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000,"confidence":0.42}]}`)
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
if len(res.Markers) != 1 {
t.Fatalf("want 1 marker, got %d", len(res.Markers))
}
if res.Markers[0].Confidence != 0.42 {
t.Errorf("confidence = %v, want 0.42 (real value, not hardcoded)", res.Markers[0].Confidence)
}
}
func TestProviderDefaultsConfidenceWhenAbsent(t *testing.T) {
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
if res.Markers[0].Confidence != defaultConfidence {
t.Errorf("confidence = %v, want default %v", res.Markers[0].Confidence, defaultConfidence)
}
}
func TestProviderPicksMostSubmittedCandidate(t *testing.T) {
body := `{"type":"episode","intro":[
{"end_ms":50000,"confidence":0.6,"submission_count":2},
{"end_ms":61000,"confidence":0.5,"submission_count":9}
]}`
p := newProvider(t, body)
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
if len(res.Markers) != 1 {
t.Fatalf("want 1 marker, got %d", len(res.Markers))
}
if got := res.Markers[0].End; got != 61*time.Second {
t.Errorf("picked end = %v, want 61s (the submission_count=9 candidate)", got)
}
if res.Markers[0].SubmissionCount != 9 {
t.Errorf("submission_count = %d, want 9", res.Markers[0].SubmissionCount)
}
}
```
- [ ] **Step 2: Run the tests to verify they fail**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/introdb/ -run TestProvider -v
```
Expected: FAIL — `TestProviderUsesRealConfidence` sees `0.9` (hardcoded), `TestProviderPicksMostSubmittedCandidate`
gets the first usable (`end=50s`, `SubmissionCount=0`). (`TestProviderResolvesTVDBOnly` already
passes from Task 2 — it guards against regression.)
- [ ] **Step 3: Rewrite `pickMarker` in `provider.go`**
Replace the entire `pickMarker` function:
```go
// pickMarker selects the best usable segment from a TheIntroDB response array.
// `requireEnd` is true for segments where the end timestamp is the load-bearing
// field (intro, recap) — they're allowed to start at 0 if `start_ms` is omitted.
// For trailing segments (credits, preview) the start is required but the end
// defaults to the file duration. When several candidates are usable (e.g. no
// duration match narrowed the set), the most-submitted one wins, with higher
// confidence breaking ties; with a single candidate this is the previous
// first-usable behavior. Real per-segment confidence is used when present,
// falling back to defaultConfidence only when the API omits it.
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
best := markers.Marker{}
bestSubs := -1
found := false
for _, s := range stamps {
start := time.Duration(0)
end := totalDuration
if s.StartMs != nil {
start = time.Duration(*s.StartMs) * time.Millisecond
}
if s.EndMs != nil {
end = time.Duration(*s.EndMs) * time.Millisecond
}
if requireEnd && s.EndMs == nil {
continue
}
if !requireEnd && s.StartMs == nil {
continue
}
if end <= start {
continue
}
confidence := defaultConfidence
if s.Confidence != nil {
confidence = *s.Confidence
}
subs := 0
if s.SubmissionCount != nil {
subs = *s.SubmissionCount
}
if !found || subs > bestSubs || (subs == bestSubs && confidence > best.Confidence) {
best = markers.Marker{Kind: kind, Start: start, End: end, Confidence: confidence, SubmissionCount: subs}
bestSubs = subs
found = true
}
}
return best, found
}
```
- [ ] **Step 4: Run the tests to verify they pass**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/introdb/ -run TestProvider -v
```
Expected: PASS (all four `TestProvider*` tests).
- [ ] **Step 5: Commit**
```bash
git add internal/markers/introdb/provider.go internal/markers/introdb/provider_test.go
git commit -m "feat(markers): use real introdb confidence and pick best candidate"
```
---
## Task 4: Full-package regression + integration build
**Files:** none (verification only)
- [ ] **Step 1: Run the entire markers test suite**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/... -v
```
Expected: PASS — the new `introdb` tests plus the existing `write_test.go` /
`types_test.go` (unchanged behavior: `BuildUpdatePayload` still aggregates max confidence,
`FetchFirstHit` still uses registration order).
- [ ] **Step 2: Build the binary to confirm nothing downstream broke**
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go build ./...
```
Expected: exit 0. (The only callers of the changed client methods are in `provider.go`, updated
in Task 2; `cmd/silo/main.go` constructs the provider but never calls the client directly.)
- [ ] **Step 3: Confirm the working tree is clean**
```bash
git status
```
Expected: clean (all changes committed in Tasks 1–3; this task adds no files).
---
## Self-review notes
- **Gap coverage:** TVDB lookups (Task 2, `TestFetch*` + `TestProviderResolvesTVDBOnly`), real
confidence (Task 3, `TestProviderUsesRealConfidence` + `…DefaultsConfidenceWhenAbsent`),
best-candidate (Task 3, `TestProviderPicksMostSubmittedCandidate`). Maps to design gaps #1–#3.
- **Containment:** the client signature change touches exactly one production caller
(`provider.go`); `markers.Marker` is constructed only in `provider.go`. Verified by grep before
planning, re-verified by the Task 4 `go build ./...`.
- **No behavior regressions:** the write path is untouched, so `BuildUpdatePayload`'s
max-confidence aggregation and its test remain valid; per-segment confidence is deferred to
Phase 2 where the merge actually consumes it.
- **No placeholders:** every code step shows the complete replacement; every run step shows the
command and the expected result. No migration, no frontend, no wiring changes.
- **Default confidence:** `defaultConfidence = 0.9` preserves today's effective value for
responses that omit confidence, so existing stored markers keep equivalent provenance.
docs/superpowers/plans/2026-06-06-marker-sources-and-contribution-implementation.md
+482
View File
@@ -0,0 +1,482 @@
# Marker sources & TheIntroDB contribution — implementation plan (all phases)
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Implement phases in order; each phase is independently shippable.
**Goal:** Implement the design in
[2026-06-06-marker-sources-and-contribution-design.md](../specs/2026-06-06-marker-sources-and-contribution-design.md):
fix the TheIntroDB read-path correctness gaps, formalize a multi-source provider model
(query-all / best-wins), and add the ability to contribute Silo's own markers back to
TheIntroDB — configured **per provider, off by default**. Server-side API + data model only;
web UI is a separate follow-up.
**Architecture:** Five sequential phases. Phase 1 fixes the existing provider (TVDB, real
confidence, best-candidate). Phase 2 adds a per-provider config table, per-segment marker
provenance, and `Registry.FetchMerged`. Phase 3 adds the submission client, the contribution
audit table, and the `ContributionService` engine. Phase 4 exposes the full admin API (manual
markers, provider config, contribute, history). Phase 5 adds the daily auto-contribution task.
**Tech Stack:** Go (chi handlers, pgx repositories), PostgreSQL (paired numbered migrations),
no frontend in this plan. Most touched packages (`internal/markers`, `internal/markers/introdb`,
`internal/taskmanager/tasks`) are pure Go; `internal/api/handlers` needs libvips (CGO image
deps). Tests run in a throwaway Go container (host has no Go toolchain).
Commands assume the repository root is the cwd.
---
## Phase overview & dependencies
| Phase | Delivers | Depends on | New migrations |
|------|----------|-----------|----------------|
| 1 | TheIntroDB read-path correctness (TVDB, real confidence, best-candidate) | — | none |
| 2 | `marker_provider_config` + per-segment provenance + `FetchMerged` | 1 (real confidence) | `marker_provider_config` |
| 3 | Submission client + `marker_contributions` + `ContributionService` | 2 (`Submitter`, provider config) | `marker_contributions` |
| 4 | Admin API: manual markers, provider config, contribute, history | 3 (service) | none |
| 5 | Daily auto-contribution task | 3 (service), 4 (optional) | none |
Phase 1's line-level steps are also captured standalone in
[2026-06-06-marker-introdb-readpath-correctness.md](2026-06-06-marker-introdb-readpath-correctness.md);
they are summarized here so this document is self-contained.
---
## Running tests (host has no Go toolchain)
Pure-Go packages (`internal/markers`, `internal/markers/introdb`, `internal/taskmanager/...`):
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
go test ./internal/markers/... -v
```
`internal/api/handlers` (Phase 4) needs libvips:
```bash
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
-e GOFLAGS=-mod=mod golang:1.26 \
sh -c 'apt-get update >/dev/null && apt-get install -y --no-install-recommends libvips-dev >/dev/null && go test ./internal/api/handlers/ -run TestAdminMarkers -v'
```
Whole-project compile: `go build ./...` (same container).
---
## Phase 1 — TheIntroDB read-path correctness
**Goal:** Honor TVDB ids, capture/use real per-segment confidence + submission_count, and pick
the best candidate among multiple. Contained to `internal/markers/introdb` plus one field on
`markers.Marker`. No migration, no write-path change.
**Files:** `internal/markers/introdb/{types.go,client.go,provider.go}` (modify),
`internal/markers/types.go` (modify, add `Marker.SubmissionCount`),
`internal/markers/introdb/{client_test.go,provider_test.go}` (create).
### Task 1.1 — Type plumbing
- [ ] Add `Confidence *float64` and `SubmissionCount *int` to `segmentTimestamps`
(`introdb/types.go`); add `const defaultConfidence = 0.9`.
- [ ] Add `SubmissionCount int` to `markers.Marker` (`markers/types.go`).
- [ ] `go build ./internal/markers/...` → exit 0. Commit.
### Task 1.2 — TVDB lookups (gap #1)
- [ ] Create `introdb/client_test.go` (httptest): asserts `tvdb_id` sent when only tvdb given,
tmdb preferred over tvdb/imdb, movie tvdb, and cache collapses repeat calls. Run → FAIL (compile).
- [ ] `client.go`: `FetchEpisode`/`FetchMovie` gain a `tvdbID` arg; query preference
`tmdb → tvdb → imdb` via a `switch`; the empty-id guard includes tvdb; `cacheKeyEpisode`/
`cacheKeyMovie` include a `tvdb:` branch.
- [ ] `provider.go` `FetchMarkers`: read `req.ExternalIDs[markers.ExternalIDKeyTVDB]`, include it
in the empty guard, pass it to both client calls.
- [ ] Run client tests → PASS. Commit.
### Task 1.3 — Real confidence + best-candidate (gaps #2, #3)
- [ ] Create `introdb/provider_test.go`: TVDB-only resolves; real confidence used; default when
absent; most-submitted candidate wins (ties broken by confidence). Run → FAIL.
- [ ] Rewrite `pickMarker` to use real confidence (fallback `defaultConfidence`), set
`SubmissionCount`, and select the candidate with the highest `(submission_count, confidence)`.
- [ ] Run → PASS. Commit.
### Task 1.4 — Regression
- [ ] `go test ./internal/markers/... && go build ./...` → all green (existing `write_test.go`/
`types_test.go` unaffected; write path untouched). Commit if needed.
(Full code for every step is in the standalone Phase 1 plan linked above.)
---
# Phase 2 — Per-provider config, per-segment provenance, FetchMerged
**Goal:** Add the `marker_provider_config` table that drives which providers are queried (and,
later, submitted to); make the marker write path carry **per-segment** provider/confidence/
algorithm so a merged multi-provider result writes correct provenance; add
`Registry.FetchMerged` (query all enabled providers, keep the best candidate per segment) and
switch the lazy-playback path to it. With only TheIntroDB enabled, behavior is unchanged.
**Files:**
- Create: `migrations/<next>_marker_provider_config.{up,down}.sql`
- Create: `internal/markers/provider_config.go`, `internal/markers/provider_config_test.go`
- Modify: `internal/markers/types.go` (`Marker.ProviderID`/`Algorithm`; `Registry.FetchMerged`)
- Modify: `internal/markers/write.go` (per-segment provenance in `MarkerUpdatePayload`/`BuildUpdatePayload`)
- Modify: `internal/scanner/file_repo.go` (`MarkerUpdate` + `applySegmentPatch` per-segment provenance)
- Modify: the `markers.MarkerUpdatePayload` → `scanner.MarkerUpdate` conversion (in the lazy-markers handler/adapter)
- Modify: `internal/api/handlers/playback_lazy_markers.go` (call `FetchMerged`)
- Modify: `cmd/silo/main.go` (construct the config store; pass to the registry)
### Task 2.1 — Migration: `marker_provider_config`
- [ ] Write `migrations/<next>_marker_provider_config.up.sql` (use the next free number; latest is 180):
```sql
CREATE TABLE public.marker_provider_config (
provider text PRIMARY KEY,
fetch_enabled boolean NOT NULL DEFAULT true,
fetch_priority integer NOT NULL DEFAULT 100,
contribute_enabled boolean NOT NULL DEFAULT false,
contribute_auto_local boolean NOT NULL DEFAULT false,
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
updated_at timestamptz NOT NULL DEFAULT now()
);
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
```
- [ ] `.down.sql`: `DROP TABLE public.marker_provider_config;`
- [ ] Commit.
### Task 2.2 — Provider config store
- [ ] Create `internal/markers/provider_config.go`:
```go
type ProviderConfig struct {
Provider string
FetchEnabled bool
FetchPriority int
ContributeEnabled bool
ContributeAutoLocal bool
ContributeMinConfidence float64
}
// ProviderConfigStore is a cached read/write façade over marker_provider_config.
// Reads serve from an in-memory snapshot; Update writes through and refreshes.
type ProviderConfigStore struct { /* pool, mu, cache map[string]ProviderConfig */ }
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore
func (s *ProviderConfigStore) Reload(ctx context.Context) error // load all rows into cache
func (s *ProviderConfigStore) List() []ProviderConfig // snapshot copy
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool)
func (s *ProviderConfigStore) Update(ctx context.Context, p ProviderConfig) error // upsert + refresh
// EnabledForFetch returns fetch_enabled providers sorted by (fetch_priority asc, provider asc).
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig
```
- [ ] Hot-reload: subscribe the store's `Reload` to the existing `cache.EventSettingsChanged`
Redis channel (mirror the `introdb.api_key` reload wiring in `cmd/silo/main.go`), or add a
dedicated `marker_provider_config_changed` event published by `Update`. Either is acceptable;
document which.
- [ ] `provider_config_test.go`: with a real/pgxmock pool or an injected fake, assert
`EnabledForFetch` filters disabled rows and sorts by priority. Commit.
### Task 2.3 — Per-segment marker provenance (write path)
This makes a merged result (intro from provider A, credits from provider B) persist the correct
per-segment `*_markers_provider` / `*_markers_confidence` / `*_markers_algorithm`. The DB columns
already exist; today `applySegmentPatch` applies one shared triple.
- [ ] Add to `markers.Marker`: `ProviderID string`, `Algorithm string` (so each segment carries
its origin). `provider.go` (Phase 1) sets these from `ProviderID`/`Algorithm` per marker.
- [ ] Rework `markers.MarkerUpdatePayload` (`write.go`) from one shared `Confidence`/`Provider`/
`Algorithm` to a per-segment struct, e.g.:
```go
type SegmentPayload struct { Start, End *float64; Provider *string; Confidence *float64; Algorithm string }
type MarkerUpdatePayload struct {
Intro, Credits, Recap, Preview SegmentPayload
Source string // shared source class (e.g. "online"); per-kind provider differs
}
```
- [ ] Update `BuildUpdatePayload` to fill each `SegmentPayload` from the matching `Marker`
(provider/confidence/algorithm per segment). Update `write_test.go`
(`TestBuildUpdatePayloadAggregatesConfidence` → assert per-segment confidence: intro 0.7,
credits 0.9, recap 0.5).
- [ ] Read `applySegmentPatch` and the `MarkerUpdatePayload → scanner.MarkerUpdate` conversion
first, then extend `scanner.MarkerUpdate` + `applySegmentPatch` so each segment's
provider/confidence/algorithm come from that segment (not a shared field). `nextSharedMarkerAttribution`
keeps populating the legacy shared `markers_source`/`markers_confidence` columns from the
highest-confidence applied segment.
- [ ] `go test ./internal/markers/... ./internal/scanner/...` → green. Commit.
### Task 2.4 — `Registry.FetchMerged`
- [ ] Give `Registry` access to the enabled set (constructor takes a `ProviderConfigStore`, or
`FetchMerged` takes an `[]ProviderConfig` snapshot). Add:
```go
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
// segment kind, the candidate with the highest (SubmissionCount, Confidence),
// breaking ties by the provider's fetch_priority. Per-segment ProviderID/
// Algorithm are preserved so the write path records correct provenance.
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error)
```
Algorithm: fan out via goroutines (bounded) to enabled providers; collect `(ProviderConfig,
Result, error)`; for each `MarkerKind`, choose the best `Marker` across results; assemble a
`Result` whose `Markers` carry their winning provider's `ProviderID`/`Algorithm`. Log and skip
provider errors (like `FetchFirstHit`). Return `ok=false` if no segment found.
- [ ] `types_test.go`: add a merge test — two fake providers, one wins intro, the other wins
credits by submission_count; assert the merged result has both with correct per-segment
ProviderID. Keep the existing `FetchFirstHit` tests. Commit.
### Task 2.5 — Switch lazy path + wire store
- [ ] In `internal/api/handlers/playback_lazy_markers.go`, replace the `FetchFirstHit` call in
`fetchOnlineMarkersForPlayback` with `FetchMerged`.
- [ ] In `cmd/silo/main.go`, construct `NewProviderConfigStore`, `Reload` it at startup, wire its
hot-reload, and pass it to the registry. (TheIntroDB stays registered as today.)
- [ ] `go build ./... && go test ./internal/markers/...` → green. With only `introdb` enabled,
`FetchMerged` returns the same result `FetchFirstHit` did — verify a provider test asserting
single-provider parity. Commit.
---
# Phase 3 — Submission client, contribution tracking, service engine
**Goal:** Build the machinery to submit a marker to a provider, idempotently and audited —
without yet wiring a trigger. Adds the `Submitter` capability, the introdb submission/stats
client methods, the `marker_contributions` table, and the `ContributionService`.
**Files:**
- Modify: `internal/markers/types.go` (`Submitter`, `SubmissionRequest`/`SubmissionResult`/`UserStats`)
- Modify: `internal/markers/introdb/{types.go,client.go,provider.go}` (submit + stats; implement `Submitter`)
- Create: `migrations/<next>_marker_contributions.{up,down}.sql`
- Create: `internal/markers/contribution_repo.go`, `internal/markers/contribute.go` (+ tests)
- Modify: `cmd/silo/main.go` (construct the service)
### Task 3.1 — `Submitter` capability + DTOs
- [ ] Add to `internal/markers/types.go`:
```go
type SubmissionRequest struct {
Kind ItemKind
ExternalIDs map[string]string // tmdb/imdb (tvdb not accepted by /submit; tmdb required)
SeasonNumber int
EpisodeNumber int
Segment MarkerKind
Start, End *time.Duration // nil start ok for intro/recap; nil end ok for credits/preview
Duration time.Duration
}
type SubmissionResult struct { ID string; Status string; Weight float64 } // status: pending|accepted|rejected
type UserStats struct { Total, Accepted, Pending, Rejected int; AcceptanceRate float64; CurrentStreak, BestStreak int }
type Submitter interface {
Provider
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
FetchUserStats(ctx context.Context) (UserStats, error)
}
```
- [ ] Commit (compiles; no implementer yet).
### Task 3.2 — introdb submission client
- [ ] `introdb/types.go`: add the request/response shapes for `POST /v3/submit`
(`tmdb_id`,`imdb_id`,`type`,`segment`,`season`,`episode`,`video_duration_ms`,`start_ms`,`end_ms`;
response `{submissions:[{id,status,weight,...}]}`) and `GET /v3/user/stats`. Remove the
"Submissions are intentionally not supported" package note.
- [ ] `introdb/client.go`: add `SubmitSegment(ctx, ...) (*submitResponse, error)` (POST,
`Authorization: Bearer` **required** — return an error if `apiKey == ""`; honor
`X-UsageLimit-Reset` on 429) and `FetchUserStats(ctx) (*userStats, error)` (GET).
- [ ] `introdb/provider.go`: implement `markers.Submitter` — map `SubmissionRequest` to the
client call (apply the v3 null rules: intro/recap drop a zero start to `null`; credits/preview
send `null` end when absent), translate `SubmissionResult`/`UserStats`.
- [ ] `client_test.go`/`provider_test.go`: httptest asserts the submit body shape, bearer header,
and that an empty key errors before any HTTP call; stats parse. Commit.
### Task 3.3 — `marker_contributions` table + repo
- [ ] Migration `<next>_marker_contributions.up.sql` (per design §C4):
```sql
CREATE TABLE public.marker_contributions (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
provider text NOT NULL,
segment_kind text NOT NULL,
source text NOT NULL,
submitted_start_ms bigint,
submitted_end_ms bigint,
video_duration_ms bigint,
content_hash text NOT NULL,
submission_id uuid,
status text NOT NULL,
http_status integer,
error text,
submitted_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (media_file_id, provider, segment_kind, content_hash)
);
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
```
- [ ] `.down.sql`: drop the table.
- [ ] `internal/markers/contribution_repo.go`: `ContributionStore` with `AlreadySubmitted(ctx,
fileID, provider, kind, contentHash) (bool, error)`, `Record(ctx, ContributionRow) error`
(upsert on the unique key), `ListByFile(ctx, fileID) ([]ContributionRow, error)`. Add a
`ContentHash(kind string, startMs, endMs, durationMs *int64) string` helper (stable SHA-256).
- [ ] Repo test (value-hash idempotency, list). Commit.
### Task 3.4 — `ContributionService`
- [ ] `internal/markers/contribute.go`:
```go
type ContributionService struct { /* registry, resolver, cfg *ProviderConfigStore, store *ContributionStore, log */ }
// ContributeFile submits the file's eligible markers to enabled submitter providers.
// opts may scope to one provider and/or specific segment kinds. Auto callers pass
// requireAutoLocal=true so only providers with contribute_auto_local participate.
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error)
```
Per provider (from `cfg.List()` filtered to `Submitter` + `contribute_enabled`, and
`contribute_auto_local` when `opts.Auto`):
1. **Eligibility (C2):** for each requested segment present on the file, require
`source ∈ {scanner, manual}` (skip `online` — circular); resolve external IDs via the existing
resolver (skip if none / tmdb required by `/submit`); for auto require `source=scanner`,
`kind=intro`, `confidence ≥ contribute_min_confidence`.
2. **Idempotency:** compute `ContentHash`; skip if `AlreadySubmitted`.
3. **Submit:** `provider.(Submitter).SubmitMarker(...)`.
4. **Persist:** `store.Record(...)` with returned status/submission_id (or `status="error"`).
- [ ] `contribute_test.go` with a fake `Submitter` + in-memory store: asserts online-sourced
markers are skipped, sub-threshold auto markers skipped, duplicates skipped, a fresh manual
marker submitted and recorded. Commit.
### Task 3.5 — Wire
- [ ] `cmd/silo/main.go`: construct `ContributionService` (registry, resolver, config store,
contribution store). Not yet called by any handler/task. `go build ./...` + tests green. Commit.
---
# Phase 4 — Admin API (manual markers, provider config, contribute, history)
**Goal:** Expose the full backend contract under `RequireAdmin`. Mirror the structure of
`internal/api/handlers/admin_intro.go` (interface-typed deps, `chi.URLParam`, `writeError`/
`writeJSON`, `MarkerUpdateNotifier` after writes).
**Files:**
- Create: `internal/api/handlers/admin_markers.go` (+ test), `internal/api/handlers/admin_marker_providers.go` (+ test)
- Modify: `internal/api/router.go` (register routes in the `RequireAdmin` group)
- Modify: `cmd/silo/main.go` (construct + inject the handlers)
### Task 4.1 — Manual marker endpoints (design §C3)
- [ ] `admin_markers.go`:
- `GET /admin/files/{fileId}/markers` → current per-segment values + provenance
(`source`,`provider`,`confidence`,`algorithm`,`detected_at`) for intro/recap/credits/preview.
- `PUT /admin/files/{fileId}/markers` → body per segment `{start,end}` (seconds) or `null` to
clear; write `source = "manual"` via `scanner.FileRepository.UpsertMarkers` (priority 4);
then `MarkerUpdateNotifier.MarkersUpdated(file)`. Validation: `end>start`; intro/recap may
omit start; credits/preview may omit end; within `[0, duration]`.
- `DELETE /admin/files/{fileId}/markers/{segment}` → clear one segment.
- `GET|PUT /admin/items/{id}/markers` → resolve to the item's primary file (convenience).
- [ ] Handler test (libvips container): PUT writes manual markers and returns provenance;
invalid range → 400; clear nulls the segment. Commit.
### Task 4.2 — Provider config + validate endpoints (design §C7)
- [ ] `admin_marker_providers.go`:
- `GET /admin/markers/providers` → for each registered provider: id, `isSubmitter`, its
`marker_provider_config` row, and (best-effort cached) `UserStats` for submitters with a key.
- `PUT /admin/markers/providers/{provider}` → update the config row (validated; unknown → 404).
- `POST /admin/markers/providers/{provider}/validate` → `Submitter.FetchUserStats`; non-submitter → 400.
- [ ] Test: list returns introdb with its config; update flips `contribute_enabled`; validate on
a non-submitter → 400. Commit.
### Task 4.3 — Contribution endpoints (design §C7)
- [ ] In `admin_markers.go` (or a small `admin_contributions.go`):
- `POST /admin/files/{fileId}/contribute` (optional body `{provider?, segments?[]}`) →
`ContributionService.ContributeFile(...)`; returns per (provider, segment) outcome.
- `GET /admin/files/{fileId}/contributions` → `ContributionStore.ListByFile`.
- [ ] Test with a fake service/store. Commit.
### Task 4.4 — Routing + wiring
- [ ] Register all new routes inside the existing `RequireAdmin` group in `internal/api/router.go`
(mirror the `redetect-intro` registration). Inject the handlers in `cmd/silo/main.go`.
- [ ] `go build ./... && go test ./internal/api/handlers/ -run 'TestAdminMarkers|TestAdminMarkerProviders'`
(libvips container) → green. Commit.
---
# Phase 5 — Daily auto-contribution task
**Goal:** A scheduled task that, for each provider with `contribute_enabled &&
contribute_auto_local`, submits high-confidence local intro detections that haven't been
contributed. Mirror `internal/taskmanager/tasks/detect_intro_markers.go`.
**Files:** Create `internal/taskmanager/tasks/contribute_markers.go` (+ test); modify
`cmd/silo/main.go` (register the task).
### Task 5.1 — `ContributeMarkersTask`
- [ ] Implement the `taskmanager.Task` interface (`Key="contribute_markers"`, `Name`,
`Description`, `Category=TaskCategoryLibrary`, `IsHidden() bool`, `DefaultTriggers()` daily e.g.
`"04:00"` — after the 03:30 detection task, `Execute`).
- [ ] `Execute`: short-circuit if no provider has `contribute_enabled && contribute_auto_local`
(read `ProviderConfigStore`). Otherwise page through episode files with
`intro_markers_source='scanner'` and `intro_markers_confidence ≥` the provider's
`contribute_min_confidence` (a new `ContributionStore`/repo query, batched), and call
`ContributionService.ContributeFile(file, ContributeOptions{Auto:true})`. Report progress; obey
the introdb usage limit (the client already backs off on `X-UsageLimit-Reset`); idempotency
makes interrupted runs resumable.
- [ ] Test with fakes: enabled+auto submits eligible files; disabled → no-op; sub-threshold
skipped. Commit.
### Task 5.2 — Register
- [ ] Register the task where `DetectIntroMarkersTask` is registered in `cmd/silo/main.go`.
- [ ] `go build ./... && go test ./internal/taskmanager/...` → green. Commit.
---
## Final integration verification
**Files:** none (build + runtime).
- [ ] **Build the image** (compiles backend + frontend):
```bash
docker build --build-arg BUILD_REVISION=$(git rev-parse --short HEAD) --build-arg BUILD_DIRTY=false -t silo-server:markers-test .
```
- [ ] **Recreate the container** (applies the new migrations on startup) and wait for health:
```bash
docker compose up -d silo
for i in $(seq 1 24); do s=$(docker inspect -f '{{.State.Health.Status}}' silo-silo-1); echo "$s"; [ "$s" = healthy ] && break; sleep 5; done
```
- [ ] **Verify migrations applied + tables exist:**
```bash
docker compose exec -T postgres psql -U silo -d silo -c \
"SELECT to_regclass('public.marker_provider_config'), to_regclass('public.marker_contributions');"
docker compose exec -T postgres psql -U silo -d silo -c "SELECT * FROM marker_provider_config;"
```
Expected: both tables present; an `introdb` row with `fetch_enabled=t, contribute_enabled=f`.
- [ ] **Smoke the read path (TVDB):** trigger playback (or call the marker fetch) for a
TVDB-only-matched episode and confirm markers now populate where they previously did not.
- [ ] **Smoke contribution (manual, dry):** enable contribution for introdb against a **test/staging**
TheIntroDB key, `PUT` a manual marker on a file, `POST /admin/files/{id}/contribute`, then
`GET /admin/files/{id}/contributions` shows a `pending` row; a second contribute is idempotent
(no new row). Use a non-production key so test data isn't submitted to the real database.
- [ ] `git status` → clean (all changes committed per phase).
---
## Self-review notes
- **Phase independence:** each phase compiles, tests, and ships alone; with only `introdb`
enabled and contribution off (defaults), Phases 2–5 are observationally inert until an operator
enables a provider — so they can land ahead of any behavior change.
- **Per-provider, off by default:** `contribute_enabled`/`contribute_auto_local` default `false`
in the migration; the service refuses to submit otherwise; auto additionally gated on
`contribute_auto_local`. The `provider` column on `marker_contributions` and the per-provider
loop make multi-target submission first-class.
- **Circular-contribution guard:** `source='online'` is never contributed (C2 eligibility) — has
a dedicated unit test in Task 3.4.
- **Idempotency:** value `content_hash` + the unique constraint mean repeat/interrupted runs
don't double-submit, while a corrected value (new hash) does submit — matching TheIntroDB's
weighted-average model.
- **Write-path accuracy:** Task 2.3 requires reading `applySegmentPatch` and the
`MarkerUpdatePayload → scanner.MarkerUpdate` conversion before editing; the per-segment columns
already exist in `media_files`, so this is a plumbing change, not a schema change.
- **Migration numbers:** `marker_provider_config` and `marker_contributions` take the next two
free numbers at implementation time (181+; latest on disk is 180). Re-check for collisions
before writing.
- **Out of scope (tracked in the design doc):** all web UI; per-user TheIntroDB accounts; a
`marker_provider.v1` plugin capability. Clean seams left for each.
docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md
+153
View File
@@ -0,0 +1,153 @@
# Audiobooks Absorption — Discovery Findings
Produced by sub-plan 1, Task 1. Locks data-model and integration
decisions for the audiobook foundation migrations and downstream sub-plans.
## D1 — Next migration number
Original snapshot: `138_search_number_word_normalization.up.sql` was the
highest existing migration. The landed implementation was renumbered to
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
`160_audiobooks_feature_flag`.
## D2 — `media_libraries` kind/type column
The spec refers to `media_libraries` but the actual table is `media_folders`.
`media_folders` has an existing column `type` (text, NOT NULL) that discriminates
library content. Current values in production: `movies`, `series`, `mixed`.
Existing column 'type' (text) discriminates library content on `media_folders`.
Task 4 (migration 159) should ADD the value `audiobooks` to the type vocabulary
rather than add a new column. The audiobook scanner branch will set
`media_folders.type = 'audiobooks'` for audiobook libraries. No schema change
needed for the column itself; migration 159 becomes a no-op DDL migration that
documents the new allowed value and adds any supporting indexes if needed.
No CHECK constraint or enum enforces the `type` column values, so adding
`audiobooks` as a value requires no DDL constraint change.
## D3 — `media_files.chapters` JSONB shape
Sample chapter JSON (live data):
[{"index": 0, "title": "Intro start", "source": "embedded", "end_seconds": 27.944, "start_seconds": 0}, {"index": 1, "title": "Intro end", "source": "embedded", "end_seconds": 1343, "start_seconds": 27.944}]
Sub-plan 2 (scanner) MUST emit objects with the same keys when writing
audiobook chapters so the existing player and serialization code accept
them without changes.
Required keys: `index` (integer), `title` (text), `source` (text),
`start_seconds` (float), `end_seconds` (float).
## D4 — `user_watch_progress` scoping (profile vs user)
user_watch_progress is profile-scoped: column 'profile_id' (text, NOT NULL, FK via
composite PK on user_id + profile_id + media_item_id). Audiobook progress slots in directly.
Additional context: the table also stores `last_file_id`, `last_resolution`,
`last_hdr`, `last_codec_video`, and `last_edition_key`. For audiobooks, only
`position_seconds`, `duration_seconds`, `completed`, and `last_file_id` are
semantically relevant; the video-specific columns (`last_resolution`,
`last_hdr`, `last_codec_video`) will be NULL for audiobook progress rows,
which is acceptable.
## D5 — `user_playback_sessions` audiobook fit
Column list:
Table "public.user_playback_sessions"
Column | Type | Collation | Nullable | Default
------------------+--------------------------+-----------+----------+---------
session_id | text | | not null |
user_id | integer | | not null |
profile_id | text | | not null |
media_file_id | integer | | not null |
play_method | text | | not null |
position_seconds | double precision | | not null | 0
is_paused | boolean | | not null | false
started_at | timestamp with time zone | | not null | now()
updated_at | timestamp with time zone | | not null | now()
Columns required by audiobook sessions: media_item_id (or equivalent),
profile/user FK, started_at, current_position_seconds (or equivalent),
status. Mark any required column as MISSING and surface in sub-plan 3.
Assessment:
- media_item_id: MISSING — table stores `media_file_id` (FK to media_files) rather
than `media_item_id`. For audiobooks, a file maps to one audiobook item, so the
item can be looked up via the file join. No schema change strictly required, but
sub-plan 3 should note this indirect join cost.
- profile_id: PRESENT (text, not null)
- user_id: PRESENT (integer, not null)
- started_at: PRESENT
- position_seconds: PRESENT (as `position_seconds`)
- status / is_paused: PRESENT (as `is_paused`); no explicit `status` enum, but
paused/playing state is representable.
- `play_method`: required for existing sessions; audiobook sessions must supply a
value (e.g. `'direct'`).
No blocking gaps. Audiobook sessions can be written to `user_playback_sessions`
without migration using `media_file_id` as the join key.
## D6 — `people` / `item_people` role conventions
item_people.role storage: `kind` smallint (NOT NULL) — NOT a text `role` column.
The column is named `kind` with type `smallint`. No CHECK constraint or enum.
Existing kind values in use (mapped from models/media.go):
1 = Actor, 2 = Director, 3 = Writer, 4 = Producer, 5 = GuestStar, 6 = Composer
(6 = Composer defined in code but 0 rows in production data)
Sub-plan 2 will UPSERT `author` and `narrator` into item_people for
audiobook items. Since the role column is an unconstrained smallint (not
a text role column and not an enum or CHECK constraint), Sub-plan 2 must:
1. Add new PersonKind constants to `internal/models/media.go`:
`PersonKindAuthor PersonKind = 7` and `PersonKindNarrator PersonKind = 8`
2. Add corresponding cases to `PersonKind.String()` returning `"Author"` and
`"Narrator"` respectively.
No migration is needed to extend a constraint — the smallint column accepts
any integer value.
## D7 — Catalog FTS handling of `type='audiobook'`
Indexes / generated columns that filter by media_items.type:
- `001_schema.up.sql`: `idx_media_items_search` — GIN on `to_tsvector('english', title || ' ' || overview)` — NO type filter, indexes ALL rows
- `001_schema.up.sql`: `idx_media_items_search_exact_title` — btree on `lower(title)` — NO type filter
- `001_schema.up.sql`: `idx_media_items_search_overview` — GIN on overview tsvector — NO type filter
- `001_schema.up.sql`: `idx_media_items_search_title_fields` — GIN on weighted title/original_title/sort_title tsvector — NO type filter (rebuilt by migrations 127 and 138)
- `001_schema.up.sql`: `idx_media_items_type_created` — btree on `(type, created_at DESC)` — indexes ALL types, used for filtering by type
- `057_calendar_indexes.up.sql`: `idx_media_items_movie_release_date` — btree WHERE `type = 'movie'` — movie-only, not FTS
- `103_media_items_last_air_date_denorm.up.sql`: `idx_media_items_last_air_date_at` — btree WHERE `type = 'series'` — series-only, not FTS
- `105_media_items_title_normalized.up.sql`: `idx_media_items_title_normalized_trgm` — gin trigram on `title_normalized` — NO type filter (rebuilt by 127 and 138)
- `138_search_number_word_normalization.up.sql`: `idx_media_items_search_title_fields` (current) — GIN on weighted tsvector — NO type filter
- `138_search_number_word_normalization.up.sql`: `idx_media_items_title_normalized_trgm` (current) — gin trigram — NO type filter
Verdict: audiobooks WILL be FTS-searchable out of the box.
All FTS indexes on `media_items` operate on the full table with no type
restriction. A row with `type = 'audiobook'` will be indexed automatically
by `idx_media_items_search_title_fields` and `idx_media_items_title_normalized_trgm`
as soon as it is inserted. No extra migration is needed for Sub-plan 3 to
extend type filters.
## D8 — First-party scheduled-task registration
First-party scheduled tasks register at: `cmd/silo/main.go:1239–1285`
Registration call shape (from existing first-party tasks):
```go
taskMgr.Register(tasks.NewSyncCollectionsTask(collectionSyncScheduler))
```
Full interface required (from `internal/taskmanager/tasks/sync_collections.go`):
- `Key() string` — unique string key e.g. `"sync_podcast_feeds"`
- `Name() string` — human-readable name
- `Description() string` — human-readable description
- `Category() taskmanager.TaskCategory` — e.g. `taskmanager.TaskCategoryLibrary`
- `IsHidden() bool`
- `DefaultTriggers() []taskmanager.TriggerConfig` — e.g. interval trigger
- `Execute(ctx context.Context, progress taskmanager.ProgressReporter) error`
Sub-plan 5 (podcasts) will register `podcastfeed.Refresher` at `cmd/silo/main.go`
in the task registration block (around line 1259–1265) using the same pattern:
```go
taskMgr.Register(tasks.NewSyncPodcastFeedsTask(podcastFeedRefresher))
```
docs/superpowers/specs/2026-05-24-audiobook-ui-redesign-design.md
+447
View File
@@ -0,0 +1,447 @@
# Audiobook UI Redesign — Design Spec
**Date:** 2026-05-24
**Branch context:** `feat/audiobooks` in `silo-server`
**Status:** Approved — ready for implementation plan
**Related spec:** [`2026-05-24-audiobooks-absorption-design.md`](./2026-05-24-audiobooks-absorption-design.md)
## Goal
Bring the audiobook detail page and audiobook player to visual and
interaction parity with Silo's existing video player, translated for
audio. Today the audiobook surfaces work but feel generic, hide useful
information, and re-implement primitives the video player already has
in a polished form.
User-visible outcome:
- The audiobook player has the same two-mode shape as the video player
(compact HUD + immersive full-screen mode), so listening can be
background or foreground depending on intent.
- The detail page foregrounds the information audiobook listeners care
about — current chapter, narrator, listened progress — instead of
burying it.
- The audiobook-only affordances that don't exist today (sleep timer,
speed menu, bookmarks, narrator emphasis) have a home.
## Hard constraints
- **Reuse video-player primitives.** `SeekBar`, `ChaptersMenu`, and the
glass-disc button visual treatment all live in `web/src/player/` and
are already wired up correctly. The audiobook player must consume
them, not re-implement them.
- **One audio element.** Switching between mini and Now Listening must
not remount the audio element or restart the stream. Both modes are
chrome layered over the same playback state.
- **No new top-level routes.** The Now Listening view is an overlay on
whatever route the user is on, not a `/audiobooks/listen` page. This
mirrors how fullscreen video works — fullscreen is a mode, not a
destination.
- **Player state must be liftable.** The split between
`useAudiobookPlayback` (state hook) and `MiniBar` / `NowListening`
(chrome) must allow a future v1.1 change to lift the hook into an
app-level provider so the mini bar can survive page navigation.
v1 itself ships with the player tied to the detail route, same as
today.
## Scope
### In
- New mini-bar layout (cover tile, chapter title, glass-disc transport,
utility rail with sleep / chapters / speed / expand / close).
- New Now Listening full-overlay mode (large cover, chapter heading,
tall seek bar, sleep / chapters / speed / bookmark / car-mode row,
remaining-time toggle, overflow menu).
- Restructured audiobook detail page (progress + chapter-aware Resume
action, chapters expanded by default with currently-playing
highlight, narrator card, embedding-based "Similar audiobooks"
rail, "Also by author", "In this series").
- Extraction of `CircleButton` and a new `SpeedMenu` into
`web/src/player/components/` as shared primitives between video and
audiobook players.
- Split of today's monolithic `AudiobookPlayer.tsx` into a state hook
(`useAudiobookPlayback`) plus two chrome components (`MiniBar`,
`NowListening`) under `web/src/pages/audiobooks/player/`.
- Sleep timer (client-side only; pauses + short fade-out when the
timer fires).
### Out (deferred to v1.1)
- **Bookmarks.** UI affordances (mini-bar button, Now Listening
utility-row button, "Bookmarks (n)" detail-page action) ship
**hidden** in v1. Full feature needs `audiobook_bookmarks` table +
CRUD endpoints, which is its own sub-spec.
- **"Also by narrator" / cross-book "In this series" rails.** Render
only when the backend already exposes the data; otherwise hide.
Server-side joins to power them are a follow-up.
- **Car mode.** Hidden in v1; v1.1 will add a huge-buttons layout
variant reached from the utility row.
- **Audiobook library page redesign.** Out of scope; current grid
stays.
- **Persistent player across page navigation as a lifted React
context.** The current detail page owns the player state; lifting it
to an app-level provider so the bar survives navigating away from
the detail page is a follow-up. v1 ships with the bar tied to the
detail route, same as today, but visually and structurally ready
for that lift.
## Architecture
### Three connected surfaces
```
detail page ──opens──► mini bar ──expand──► Now Listening
▲ │
└─────── collapse ──────┘
```
- **Detail page** (`/audiobooks/book/:id`) is the landing surface.
Stays mounted while the player is open.
- **Mini bar** is a fixed bottom strip (current behavior) that appears
once the user clicks Play/Resume.
- **Now Listening** is a full-viewport overlay (z-index above the mini
bar, below modals) reached by clicking the cover-art tile or the
expand chevron in the mini bar. Dismissed via a collapse chevron.
The mini ↔ Now Listening transition uses a CSS view transition keyed
on the cover art element so the small tile appears to grow into the
big cover. The same primitive already used by `ViewTransitionLink`.
### Shared player primitives
`web/src/player/components/` already houses the visual building blocks
used by the video player. Two of them get extracted so both players
consume the same source of truth:
| File | Status | Purpose |
|---|---|---|
| `SeekBar.tsx` | reused as-is | already shared |
| `ChaptersMenu.tsx` | reused as-is | already shared |
| `CircleButton.tsx` | **new** — extracted from `PlayerControls.tsx` | the glass-disc primary/secondary button used in both transport clusters |
| `SpeedMenu.tsx` | **new** | popover speed menu matching the styling of `ChaptersMenu`; replaces the raw `<select>` today; video player can adopt later |
| `SleepTimerMenu.tsx` | **new** | popover with off / 5 / 15 / 30 / 45 / 60 / end-of-chapter; audiobook-only but lives here for symmetry |
Extracting `CircleButton` removes a divergence rather than adding a
layer — it's the kind of focused improvement that earns its keep
because today the video player and the audiobook player each render
their own visually-similar-but-not-identical disc buttons.
### Audiobook player component tree
```
web/src/pages/audiobooks/
AudiobookDetail.tsx (restructured per "Detail page" below)
AudiobookLibrary.tsx (unchanged)
player/ (new folder)
AudiobookPlayer.tsx (top level: owns audio element and state)
MiniBar.tsx (compact chrome)
NowListening.tsx (overlay chrome)
CoverExpandTile.tsx (left-edge tile inside MiniBar — also the view-transition anchor)
useAudiobookPlayback.ts (state hook: audio events, progress reporting, sleep timer)
```
`AudiobookPlayer` becomes a thin shell:
```tsx
function AudiobookPlayer(props) {
const playback = useAudiobookPlayback(props);
const [mode, setMode] = useState<"mini" | "now-listening">("mini");
return (
<>
<audio ref={playback.audioRef} src={playback.streamUrl} preload="metadata" hidden />
{mode === "mini"
? <MiniBar playback={playback} onExpand={() => setMode("now-listening")} onClose={props.onClose} />
: <NowListening playback={playback} onCollapse={() => setMode("mini")} />}
</>
);
}
```
The audio element lives in the parent so a mode swap never touches it.
### useAudiobookPlayback responsibilities
Lifts today's monolithic `AudiobookPlayer.tsx` into a single hook
returning a stable shape:
```ts
{
audioRef, // ref<HTMLAudioElement>
streamUrl, // string
playing, currentTime, duration, buffered, rate,
chapters, // PlayerChapter[] (flattened across files)
currentChapter, // PlayerChapter | null
sleep, // { remainingMs: number | null, end: SleepEndCondition | null }
togglePlay, seekTo, skip,
setRate,
setSleep, // arms the timer
// bookmarks (v1.1 — initially returns []/no-op stubs)
bookmarks, addBookmark, removeBookmark,
}
```
The hook owns: audio event wiring, periodic progress reporting (the
existing 10s `useReportAudiobookProgress` cadence), sleep-timer state
+ fade-out, computing `currentChapter` from `currentTime`, and the
unmount-time pause + final report. The chrome components stay pure
presentational.
## Mini bar
Layout (one seek-bar row above a controls row):
```
[ full-width SeekBar with chapter ticks ]
[ cover tile | title + chapter + time | ◀30 ▶❚❚ 30▶ | ⌛ ☰ 1× ⌃ ✕ ]
↑ left col middle center right rail
36×54px truncating shared CircleButton cluster
sleep, chapters, speed,
expand chevron, close
(bookmark hidden in v1)
```
Differences from today's implementation:
- **CoverExpandTile** added at the left edge. 36×54 (portrait
aspect-2:3). On hover: subtle scale + a small "expand" chevron
overlay. Clicking it triggers the view transition to Now Listening.
- **Left text column** shows title (line 1) and current chapter title
(line 2). Time row stays where it is. Today the chapter title isn't
shown anywhere while playing — this is the single most useful
"where am I" cue and it's missing.
- **Center cluster** swaps the bespoke `CircleButton` defined inline
in the current `AudiobookPlayer` for the extracted shared one. Same
three controls (back 30 / play-pause / forward 30).
- **Right rail** replaces the raw `<select>` for speed with
`SpeedMenu` (popover, same styling as `ChaptersMenu`). Adds
`SleepTimerMenu`, an expand-up chevron (`ChevronUp`), and the
existing close `X`. The bookmark button is **hidden in v1** (mini
bar has no bookmark control until v1.1 lands).
Mini bar height is unchanged from today.
## Now Listening overlay
Full-viewport overlay, dark surface (`bg-background`). Z-index above
the mini bar, below modals.
Layout on desktop (width ≥ md):
```
┌─────────────────────────────────────────────────────────────────┐
│ ⌄ (collapse, top-left) ⋯ More (top-right) │
│ │
│ ┌──────────────────────┐ │
│ │ │ PROJECT HAIL MARY │
│ │ [ COVER ] │ Andy Weir │
│ │ ~360 × 540 │ Narrated by Ray Porter │
│ │ │ │
│ │ │ ── CHAPTER 7 ── │
│ │ │ The Astrophage │
│ └──────────────────────┘ │
│ │
│ [ tall SeekBar with chapter ticks ] │
│ 12:43 -4:08:35 │
│ │
│ ◀30 ▶❚❚ (large) 30▶ │
│ │
│ ⌛ Sleep ☰ Chapters (24) 1× Speed │
└─────────────────────────────────────────────────────────────────┘
```
Layout on mobile (portrait): same elements stacked top-to-bottom —
header row, cover (centered, smaller), metadata block, seek bar,
transport, utility row. No horizontal split.
Key behaviors:
- **Tap right time** toggles between `-remaining` and the wall-clock
end time. The toggle preference is in-memory only (resets per
session) — no persistence concern.
- **Sleep timer** opens `SleepTimerMenu`. When armed, the trigger
shows the countdown (`Sleep 04:32`). When the timer hits zero the
player fades volume to 0 over 5s and pauses.
- **Car mode** and **bookmark** buttons are **hidden in v1** — the
utility row ships with sleep / chapters / speed only. Car mode +
bookmark slots are documented here so v1.1 has a clear home for
them.
- **Overflow ⋯** menu items for v1: "Go to detail page", "Report a
sync problem" (opens a mailto/issue link). Future additions live
here.
The same `<audio>` element backs both modes, so all controls in Now
Listening are bound to the same `useAudiobookPlayback` state.
## Detail page restructure
The page keeps its current top-level shape (hero band + content
below) and adds/rearranges these blocks:
### Hero band
Today: `DetailHero` with a stack of `Play` / `Play from Start`
actions and a small progress bar that appears below the buttons only
when progress exists.
New version (still using `DetailHero` but with a richer `actions`
slot):
```
[ progress bar — full width of actions column, always shown if any progress ]
[ "3h 12m listened · 19%" caption ]
▶ Resume Ch 7 · The Astrophage ↻ Start Over 🔖 Bookmarks (3)
```
- Progress bar moves up *above* the buttons so the listener sees
position context first.
- Resume button label includes the chapter the listener will land
in. Pulled from the same `buildChapterList` already in the file.
- "Bookmarks (n)" button visible only when n > 0; opens a side sheet
listing them. v1 ships with an empty stub (always 0, button never
shown) — full feature in v1.1.
### Chapters section
Today: collapsed-by-default disclosure showing a chapter list.
New version:
- Expanded by default. (Collapsed in this position made sense when
the page was sparse; once we're foregrounding chapter awareness
everywhere else, hiding the list here is contradictory.)
- Currently-playing chapter row gets a left-edge accent and a
"▶ playing" badge on the right.
- Lightweight sort menu in the section header: "By position" (default),
"Longest first" — no UI for "By title" since chapter titles rarely
sort meaningfully. Sort preference is in-memory only for v1.
- Clicking a chapter row still calls `openPlayer(absoluteStart)` as
today; if the player is open, it seeks rather than reopens.
### Narrator card
New section between Chapters and the cross-rails. Renders only when
`data.narrator` is non-empty.
```
── NARRATOR ──
[ avatar 64×64 ] Ray Porter
47 audiobooks in your library →
```
Avatar source: if the absorbed audiobook data model exposes a
narrator photo, use it; otherwise an initials avatar in a circle.
The `→` link goes to a narrator detail page if one exists, or a
search-results page filtered by narrator name as a fallback.
### Cross-book rails
Three sections, each rendered only when its data is present, ordered
top-to-bottom:
- **"Similar audiobooks"** — embedding-based recommendations. Ranked
by vector similarity against this book; ordering and inclusion
decided server-side. Renders with a subtitle "Based on listening
patterns" to give the rail a small bit of provenance copy that the
metadata rails don't need.
- **"Also by {author}"** — horizontal scroller of cover tiles, same
pattern as `MediaRow` used elsewhere in Silo.
- **"In this series"** — same pattern, with "Book n of m" subtitle and
the current book non-clickable / highlighted.
All three rails ship behind feature-detection: if the API response
for the book includes the related array, render it; if not, hide the
section without showing a placeholder. This avoids blocking v1 on any
specific backend rail support — each lights up as the data lands.
### What's *not* in the new detail page
- Reviews / ratings — out of scope, not in the data model.
- Tabs — page is short enough that one scroll surface beats tabs.
- Social / cohost listening — no signal it's wanted.
## Data & API impact
Most of the work is pure frontend, but three small backend-adjacent
items to call out:
| Item | Impact |
|---|---|
| Sleep timer | Frontend-only. No API change. |
| Speed menu | Frontend-only — already a client-side audio setting. |
| Bookmarks | **Deferred to v1.1.** Needs `audiobook_bookmarks` table (`content_id`, `profile_id`, `position_seconds`, `note`, `created_at`) plus CRUD endpoints. v1 ships with the button hidden and `useAudiobookPlayback` exposing no-op bookmark stubs. |
| Narrator card with "n audiobooks in library" count | Needs narrator-aware query. If the detail endpoint already returns it, render; otherwise hide the count line and show only the name. |
| "Also by author" / "In this series" rails | Render only when arrays present. v1 doesn't require backend changes — it just adapts to whatever the response contains. |
| "Similar audiobooks" rail (embedding-based) | Render only when `similar_audiobooks` array present on the detail response. Backend computes similarity from per-book embeddings server-side and includes the top-N already ranked. v1 frontend has no embedding logic — it just renders whatever ordered list the API returns. |
## Routing & view-transition wiring
- `/audiobooks/book/:contentId` continues to be the only audiobook
route added by this work. Mini bar and Now Listening are layered
inside this route's component tree.
- View transition name: `audiobook-cover-{contentId}`. Set on the
cover tile inside `CoverExpandTile` and on the large cover element
inside `NowListening`. Same `contentId` on both = the browser
animates the rect change automatically.
- The detail page's hero cover and the mini bar's cover tile share
the same view-transition name so the *initial* open (clicking
Play/Resume on the hero) also animates the cover into the mini bar
position.
## Accessibility
- All transport controls have `aria-label`s (already true today; keep
parity).
- The expand chevron and the cover tile are both labeled "Expand
player" / "Open Now Listening" — two ways in, both discoverable to
screen readers.
- `SleepTimerMenu`, `SpeedMenu`, `ChaptersMenu` use the existing
`Popover` primitive with proper roving focus + Escape-to-close
(inherits from `ChaptersMenu` behavior).
- Now Listening overlay traps focus while open and restores focus to
the mini bar's expand chevron on close.
- Color contrast for the chapter-title text in the mini bar must
meet WCAG AA against the bar's background (use `text-foreground`
with a `text-muted-foreground` chapter title is the existing
pattern and passes).
## Testing
- **Unit:** `useAudiobookPlayback` — chapter computation from current
time across multi-file audiobooks, sleep-timer arm/disarm/fire,
progress reporting cadence and pause/seek/end triggers.
- **Component:** `MiniBar` and `NowListening` render expected
elements given a mocked playback object; expand/collapse calls the
right callbacks; speed menu / sleep menu open and emit the right
values.
- **Integration / Playwright** (if the repo has Playwright wired for
these flows): start playback from detail page → mini bar appears
with chapter title → click cover tile → Now Listening overlay
appears → seek bar works in both modes → collapse returns to mini
bar with playback uninterrupted (key assertion: `audio.currentTime`
monotonically advances across the mode swap).
## Out of scope (recap)
- Bookmarks backend (v1.1 sub-spec)
- Cross-book rails backend joins
- Car mode full layout
- Audiobook library grid redesign
- Lifting the player to an app-level provider so it survives
navigation away from the detail page
## Open questions / risks
- **Multi-file audiobooks.** Today's player only plays `files[0]` —
multi-file audiobooks display all chapters but can't actually cross
file boundaries. This redesign doesn't fix that; it inherits the
limitation. Worth flagging in the implementation plan whether to
fold a fix into v1 or leave it for a separate ticket.
- **View transition browser support.** CSS view transitions are
supported in modern Chromium and Safari Tech Preview; Firefox is
not there yet. The mini ↔ Now Listening swap must work without
the transition (graceful fallback to a plain unmount/mount).
- **Sleep-timer fade.** A 5s gain ramp on `HTMLAudioElement.volume`
works but is not as smooth as a Web Audio API gain node. Starting
simple; revisit if it feels janky.
docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md
+382
View File
@@ -0,0 +1,382 @@
# Audiobooks Absorption — Design Spec
**Date:** 2026-05-24
**Branch context:** `feat/audiobooks` in `silo-server`
**Status:** Approved — ready for implementation plan
## Goal
Absorb the `silo-plugin-audiobooks` plugin into the primary `silo-server`
Go application. Eliminate it as a plugin. User-visible outcome:
- Silo's existing web UI gains audiobook + podcast browse/playback.
- Audiobookshelf-compatible (ABS) mobile/desktop clients connect to silo
directly and play audiobooks from libraries silo already scans.
## Hard constraints
- **Minimize changes.** Reuse silo-server's existing tables and infrastructure
wherever they fit. Add new tables only where there is no existing equivalent.
- **No separate SPA.** Audiobook UI lives inside silo's existing React app at
`web/src/`. The plugin's standalone SPA is dropped.
- **Audiobook files = scanned by silo.** Silo's existing scanner discovers
audiobook files in configured library paths the same way it scans movies
and TV. The `silo-plugin-local-audiobooks` plugin is no longer needed.
- **No external backends.** BookWarehouse / other audiobook backend plugins
are out of scope. Local filesystem is the only source.
- **No standalone HTTP listener.** Everything runs on silo's main `:8080`
listener, including ABS Socket.io. The plugin's standalone-listener
workaround existed only because the host plugin-proxy could not bridge
WebSocket upgrades; that problem disappears once code runs in-process.
## Scope
### In
- Audiobook entity, library, playback (silo SPA + ABS clients).
- ABS-compatible REST API + Socket.io realtime channel.
- Podcasts: subscribed RSS feeds with episode refresh, plus filesystem podcasts.
- Per-profile listening progress, active play sessions, basic chapter navigation.
### Out
- Audiobook requests flow (`silo-plugin-audiobook-requests` untouched).
- Smart collections, share links, content restrictions, metadata-provider integration.
- Embedding-powered "similar books" recommender.
- External enrichment through the audiobook metadata plugin/provider — deferred;
scanner extracts local tags and identifiers for v1.
- Group listening / cowatch parity for audiobooks (silo already has cowatch for
video; revisit later if needed).
## Architecture
### Package layout
New top-level Go package: `internal/audiobooks/`.
```text
internal/audiobooks/
abs/ ← ported from plugin's internal/abs/ (~6.8k LOC)
ABS-compatible REST handlers, mounted under /abs/* and the
legacy /api/* paths ABS clients hardcode.
abssocket/ ← ported from plugin's internal/abssocket/ (~250 LOC)
Socket.io protocol on top of gorilla/websocket; optional
Redis pub/sub adapter for multi-replica fan-out.
podcastfeed/ ← ported from plugin's internal/podcastfeed/ (~350 LOC)
RSS feed refresher; registered as a scheduled task.
service.go ← thin orchestrator wiring the above to silo's existing
catalog/playback/session/auth services.
```
### Wiring touchpoints in existing silo code
| Existing file | Change |
|---|---|
| `internal/api/router.go` | Mount silo-native audiobook routes under `/api/v1/audiobooks/*`; mount ABS routes under `/abs/*` plus the small set of legacy `/api/*` paths ABS clients hit; mount the `/abs/socket.io/` endpoint. |
| `internal/scanner/` | Per-library dispatch on `media_libraries.kind`. New parsers for `audiobooks` and `podcasts`. |
| `cmd/silo/main.go` (task wiring) | Register `podcastfeed.Refresher` as a scheduled task. |
| `internal/playback/` | Accept `media_items.type='audiobook'` and `'podcast_episode'` (mostly already generic). |
| `web/src/pages/`, `web/src/player/` | New audiobook/podcast pages and player component. |
### What the plugin contributed that is dropped
The plugin's other packages — `server`, `store`, `enrich`, `recommend`,
`smartcoll`, `event`, `consumer`, `libsync`, `migrate`, `runtime`,
`bookref`, `cdn`, `mediatoken`, `streaming` — are either dropped (out-of-scope
features) or replaced by reuse of silo's existing equivalents (catalog store,
media-token signing, CDN, streaming, event bus, migrations runner).
### Total code budget
- ~7.4k LOC ported from the plugin.
- ~1.5–2k LOC new TypeScript/TSX in silo's SPA.
- ~400 LOC of new Go glue (scanner branches, router wiring, service.go).
- Three new SQL migrations in the current stack (`147_abs_sessions`,
`157_podcast_feeds`, `160_audiobooks_feature_flag`) plus one documented
no-op migration for the already-present media-folder type column.
## Data model
### Reused silo tables (no schema changes)
| Audiobook concept | Silo table | Notes |
|---|---|---|
| Audiobook entity | `media_items` | New value: `type='audiobook'`. Existing title/year/overview/poster/sort_title cover the basics. |
| Podcast (show) | `media_items` | New value: `type='podcast'`. |
| Podcast episode | `episodes` | Parallel to TV episodes; FK to parent `media_items` row. |
| Audio file + chapters | `media_files` | The `chapters jsonb` column from migration 066 stores chapters exactly as silo already does for video. |
| Library scope | `media_libraries` + `media_folders` + `media_item_libraries` | Audiobook libraries are libraries with `kind='audiobooks'` (see below). |
| Author / narrator | `people` + `item_people` | Two new role string constants: `'author'`, `'narrator'`. |
| Listening position | `user_watch_progress` | Per-profile position in file. Naming reads as video-only but works fine for audio. |
| Active play session | `user_playback_sessions` | Generic enough to host audiobook sessions. |
| Generic auth | `auth_sessions` | Used by silo's own clients. ABS clients use the new `abs_sessions` table. |
| Series / author shelves | `library_collections` + `library_collection_items` | Reuse for audiobook series, "by author" collections. |
### New tables (two migrations)
1. **`abs_sessions`** (migration 147) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token hash, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token_hash`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
2. **`podcast_feeds`** (migration 157) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
### Schema-touch column add
- `media_libraries.kind` — `'movies' | 'tv' | 'audiobooks' | 'podcasts'`. If a
similar column already exists on `media_libraries`, reuse it; otherwise this
is a third small migration. The scanner reads it to pick the per-library
parser.
### Plugin migrations NOT ported
Smart collections, share links, embeddings, content restrictions,
metadata-provider integration, request provider, file cache, listening stats aggregates,
reading goals, notification prefs, standalone-mode tables, recommender
embeddings. All represent features dropped from scope.
### Data migration
None. The plugin's `audiobooks.*` Postgres schema is left intact until cutover
is confirmed, then dropped. Silo's scanner repopulates from the filesystem on
first run. ABS clients re-authenticate once (their tokens lived in the plugin
DB, not silo's). This is the one user-visible cost.
## Routing & ABS-compatibility
All routes mount on silo's main `:8080` HTTP listener.
### 1. Silo-native audiobook API
- Prefix: `/api/v1/audiobooks/*`.
- Examples: `GET /api/v1/audiobooks/library/{id}/items`,
`GET /api/v1/audiobooks/{item_id}`,
`POST /api/v1/audiobooks/{item_id}/progress`.
- Mounted next to other v1 routes. Standard silo auth middleware. JSON shapes
follow silo's existing patterns — not the ABS shape.
### 2. ABS-compatible API
- Prefix: `/abs/*` for the bulk of client-facing endpoints, plus the small set
of `/api/*` paths ABS clients hardcode (e.g. `/api/libraries`, `/api/me`,
`/api/items/{id}`). To avoid colliding with silo's existing `/api/v1/*`,
ABS gets its full namespace at `/abs/`; ABS-legacy `/api/*` paths are
registered explicitly and scoped to ABS auth.
- Auth: a separate middleware `audiobooks.RequireABSSession` validates the ABS
bearer token against `abs_sessions`.
- Login: `POST /abs/login` accepts ABS-shaped credentials, internally calls
silo's existing auth backend (same code path as `POST /api/v1/auth/login`),
then issues an ABS token bound to a row in `abs_sessions`. No password
handling lives in audiobooks code itself.
### 3. ABS Socket.io endpoint
- Path: `/abs/socket.io/`.
- Uses silo's existing `gorilla/websocket` upgrader. The ported `abssocket/`
package handles Socket.io handshake, framing, and the long-polling fallback.
- Multi-replica fan-out: if `REDIS_URL` is set, use a Redis pub/sub adapter
(already implemented in plugin). Unset → in-memory single-replica.
### What goes away
- No standalone listener — main `:8080` only.
- No `SILO_HOST_URL` / `SILO_PLUGIN_TOKEN` env plumbing — those existed because
the plugin called back into silo over HTTP. Direct Go calls replace this.
- No host plugin-proxy in the request path.
### Streaming reuse
ABS stream URLs internally rewrite to silo's existing
`/api/v1/stream/{session_id}` machinery — direct play, range requests,
transcode fallback if needed, media-token signing. **Zero new transcode or
stream code.**
## Scanner integration
### Per-library dispatch (~30 LOC in `internal/scanner`)
The scanner reads `media_libraries.kind` and picks a parser. Existing
`movies`/`tv` paths are unchanged. New branches: `audiobooks`, `podcasts`.
### Audiobook parser (~200 LOC, `internal/scanner/audiobook.go`)
- Folder convention: one audiobook = one folder. Files = chapters in filename
order, or one `.m4b` with embedded chapters.
- Recognized extensions: `.m4b`, `.mp3`, `.m4a`, `.flac`, `.opus`.
- Single `.m4b`: one `media_items` row + one `media_files` row. Embedded
chapters extracted via the existing ffprobe call site → `media_files.chapters`.
- Multi-file folder: one `media_items` row + N `media_files` rows. Chapter
index = filename ordering; each file's `chapters` carries one synthesized
chapter for that file.
- Metadata: ID3 / MP4 tags (title, author, narrator, series, year, cover).
Reuses silo's existing tag-extraction helpers.
### Podcast parser (~150 LOC, `internal/scanner/podcast.go`)
Two paths:
1. **Filesystem podcasts** (downloaded episodes on disk): folder = podcast,
file = episode. Same `series → episodes` shape silo already handles for TV.
2. **RSS-subscribed podcasts**: no filesystem walk. The `podcastfeed.Refresher`
scheduled task fetches RSS, upserts `media_items` (podcast) + `episodes`,
optionally downloads enclosures to a configured cache dir (reusing silo's
existing download/cache infrastructure). `media_files` rows point at the
cached file once downloaded.
### Author / narrator extraction
Tag-derived names → upsert into `people` → link via `item_people` with roles
`'author'` / `'narrator'`. Same upsert pattern silo already uses for actors
and directors. No new code, just two more role string constants.
### External enrichment
Deferred. The scanner extracts local tags and identity hints only. Canonical
enrichment remains the responsibility of the audiobook metadata plugin/provider
path and is not ported in this foundation pass.
### Scheduled task
`podcastfeed.Refresher` runs every 10 minutes (matches plugin behavior).
Registered with silo's existing task manager. No other new background jobs.
### Impact on existing scanner behavior
None for movie/TV libraries. Audiobook/podcast libraries were not being
scanned before; now they are when `kind` is set appropriately.
## Web UI surfaces
### Stack
Silo's existing React 19 + Vite SPA at `web/src/`. React Query, radix-ui,
tailwind. The plugin's SPA uses the same stack so component idioms transfer
cleanly, but we are writing fresh silo pages — the plugin's `web/` is dropped.
### New pages under `web/src/pages/audiobooks/`
| Route | Purpose |
|---|---|
| `/audiobooks` | Home: continue-listening shelf, recent additions, library-wide browse. |
| `/audiobooks/library/:id` | Library view: grid of audiobooks, filter/sort, paginated. |
| `/audiobooks/book/:id` | Detail: cover, author, narrator, series, chapter list, play/continue CTA. |
| `/audiobooks/authors`, `/audiobooks/series` | Index pages built on `library_collections`. |
| `/podcasts` | Subscribed podcasts grid. |
| `/podcasts/show/:id` | Podcast detail + episode list. |
| `/podcasts/episode/:id` | Episode detail + play. |
### Player
`web/src/player/AudiobookPlayer.tsx`:
- HTML5 `<audio>` (no HLS / transcoding needed for typical audio). Falls back
to silo's existing transcode flow only if the file needs it.
- Chapter list panel, sleep timer, playback rate (0.5×–3×), 30-second seek
buttons, skip-silence toggle.
- Position updates via `POST /api/v1/audiobooks/{id}/progress` at 5–10s
intervals + on pause/seek. Mirrors silo's existing video progress cadence.
### Navigation
Add "Audiobooks" and "Podcasts" to the existing sidebar/library switcher,
driven by `media_libraries.kind`. Same dispatch point silo uses today for
Movies vs TV.
### Reused silo components (no copies)
Card grids, library shelves, search box, profile/auth chrome, image cache,
virtualized list (`@tanstack/react-virtual` already in `package.json`).
### Not building (out of scope)
Smart collections UI, share-link manager, request submission UI, embeddings
"similar books", admin enrich button, content-restriction admin.
### Frontend volume
Roughly 8 new page components + 1 player + a handful of audiobook-specific
hooks/types under `web/src/hooks/audiobooks/` and `web/src/lib/audiobooks/`.
~1.5–2k LOC of new TS/TSX total. No new top-level dependencies beyond what
silo's `package.json` already has.
## Plugin retirement & rollout
### Sibling repos
| Repo | Action |
|---|---|
| `silo-plugin-audiobooks` | Archive on GitHub; remove from catalog. |
| `silo-plugin-local-audiobooks` | Archive (silo scanner replaces it). |
| `silo-plugin-bookwarehouse-audio` | Archive (out of scope). |
| `silo-plugin-audiobook-requests` | Untouched — request flow out of scope. |
Stance: **archive, do not delete.** Leaves a recoverable home for any feature
dropped from this port that might come back later.
### Catalog update
One-line PR in `silo-plugins`: remove the three audiobook-related plugin
entries from `manifest.json`. The fourth (audiobook-requests) stays.
### Rollout sequence
1. Land the absorbed code in silo-server behind a feature-flag server setting
(`audiobooks.enabled`, default off).
2. Side-by-side: silo-server has audiobooks compiled in, the plugin is still
installed and running on this host. Verify silo's flow end-to-end
(scanner → SPA → ABS clients).
3. Stop the plugin runtime. Flip `audiobooks.enabled` on. ABS clients
reconnect to silo's `/abs/*` endpoints (config change in the ABS app if
the base URL differs).
4. Remove plugin entries from catalog; archive repos.
5. After confidence in cutover, `DROP SCHEMA audiobooks CASCADE` in Postgres.
### Rollback
Revert the silo flag. Until step 4 ships, the plugin is still installed and
runnable — single-flag rollback.
### Data migration
None. Scanner repopulates from filesystem on first run. ABS clients
re-authenticate once. This is the one user-visible cost.
## Testing
- **Unit tests** for the ported `abs/`, `abssocket/`, `podcastfeed/` packages
carry over from the plugin where they apply; rewrite tests that depended on
the plugin's separate schema.
- **Integration tests** against the real silo Postgres (per project preference
to avoid mocking the DB):
- Scanner: audiobook folder → `media_items` + `media_files` + chapters.
- ABS auth: `POST /abs/login` issues a token rooted in silo's `users`.
- ABS playback: GET a library → GET an item → stream a chapter → progress upsert.
- Podcast refresher: feed URL → upserted episodes.
- **Manual E2E**: an ABS mobile client points at silo, browses, plays, scrubs
through chapters, sleep-timer expires; silo SPA shows the listening
progress and resumes from the right chapter.
## Risks & open questions
- **Socket.io protocol port quality.** The plugin's `abssocket/` is ~250 LOC of
Socket.io framing on top of gorilla/websocket. It works today for the
plugin; once mounted on silo's main listener, verify the long-polling
fallback path still works under silo's middleware stack (auth, rate-limit,
request-id).
- **`media_libraries.kind` column.** If the existing schema already has an
equivalent column under a different name, use that instead. Verify before
writing migration 134/135.
- **`media_files.chapters` shape compatibility.** Silo's chapter JSONB shape
was designed for video chapters. Confirm the audiobook ffprobe output maps
cleanly into the same shape, or extend the JSON tolerantly.
- **Search.** The plugin had its own search index for books/authors. Silo's
existing FTS (catalog/pg-fts) needs to index audiobooks too; verify that
setting `type='audiobook'` is enough or whether the FTS index needs a small
config tweak.
- **Profile-scoped vs user-scoped state.** `user_watch_progress` is keyed —
confirm it's keyed on profile, not just user (CLAUDE.md notes silo
separates login accounts from household profiles).
## Next step
Hand off to the `writing-plans` skill to produce a phased implementation plan
covering migrations, package ports, router wiring, scanner branches, SPA
pages, and cutover. The plan's first phase should resolve the open questions
in the **Risks** section before code lands.
docs/superpowers/specs/2026-05-26-abs-bookmarks-design.md
+262
View File
@@ -0,0 +1,262 @@
# ABS Bookmarks — Phase 1 Sub-Project 1
**Status:** Approved 2026-05-26. Ready for implementation plan.
**Scope:** First sub-project of Phase 1 (per `2026-05-26-abs-implementation-fix-design.md`).
**Predecessor spec:** `docs/superpowers/specs/2026-05-26-abs-implementation-fix-design.md` §"Phase 1 — Feature surface completion" → Bookmarks bullet.
Commands in this document assume the repository root is the cwd.
## 1. Goal
Let an Audiobookshelf mobile client (official Android, official iOS, and well-behaved third parties such as Plappa) create, edit, and delete in-book bookmarks against a silo audiobook library, and see those bookmarks sync in real time to the user's other connected devices.
## 2. Non-goals
- No ebook bookmarks (audiobook only — matches ABS's own surface).
- No bulk bookmark import / export, no merge-by-title, no fuzzy time matching.
- No `/api/me/bookmarks` aggregate-list endpoint. The Android client only calls per-item endpoints; adding the aggregate would be dead code today.
- No client UI work — silo's web admin doesn't ship a bookmarks view yet, and the mobile clients already render bookmarks from these endpoints.
- No socket-server overhaul; Phase 1 just publishes events through the existing `Handler.publish` wrapper. Phase 2 covers the full socket event surface.
## 3. Architecture
Three ABS-compatible REST handlers + one socket event family + one migration.
- **HTTP layer:** new `internal/audiobooks/abs/bookmarks_handler.go` alongside existing `items_handler.go` / `progress.go`. Handler-method pattern mirrors `handleSetItemProgress`. Routes registered under both `/abs/api` and `/api` prefixes inside the existing bearerAuth group.
- **Storage layer:** new `BookmarkStore` interface in `internal/audiobooks/abs/bookmarks.go` (split out rather than crowd `progress.go`) plus a concrete `ABSBookmarkStore` Postgres implementation in `internal/audiobooks/abs_bookmark_store.go` — naming mirrors `abs_playback_session_store.go`. Wiring lives in `internal/audiobooks/service.go`, alongside the other store constructions in `BuildABSHandler`.
- **Migration:** `migrations/148_abs_bookmarks.up.sql` + `.down.sql`.
- **Socket events:** the existing nil-safe `Handler.publish(userID, event, payload)` wrapper. Each handler fires `user_updated` with a `reason` discriminator on success — three lines per handler. No new event publisher code.
## 4. Endpoint surface
All mounted at both `/abs/api/*` and `/api/*` inside the bearerAuth group.
| Verb | Path | Body | Returns |
|---|---|---|---|
| POST | `/me/item/{itemId}/bookmark` | `{title, time}` | full item bookmark list |
| PATCH | `/me/item/{itemId}/bookmark` | `{title, time}` | full item bookmark list (upserts at `time`) |
| DELETE | `/me/item/{itemId}/bookmark/{time}` | — | full item bookmark list (idempotent) |
All three handlers fire `user_updated` on success with the relevant `reason`:
| Handler | `reason` value |
|---|---|
| Create | `bookmark_created` |
| Update | `bookmark_updated` |
| Delete | `bookmark_deleted` |
Socket event payload shape:
```json
{
"reason": "bookmark_created",
"bookmark": { "id": "01K…", "libraryItemId": "126887…", "time": 1234.5, "title": "Cliffhanger", "createdAt": 1779786284823, "updatedAt": 1779786284823 }
}
```
For `bookmark_deleted`, `bookmark.title` is the pre-delete title; for `bookmark_created` / `bookmark_updated` it is the freshly-persisted row.
### 4.1 Wire shape
```json
{
"id": "01KSHR…",
"libraryItemId": "126887440911695876",
"time": 1234.5,
"title": "Good cliffhanger",
"createdAt": 1779786284823,
"updatedAt": 1779786284823
}
```
- All six fields always present (never `omitempty`).
- `time` is fractional seconds (`double`).
- `createdAt` / `updatedAt` are JS-epoch milliseconds (`int64`), matching every other ABS handler's timestamp emission.
- Empty title serializes as `"title": ""`.
### 4.2 Why no GET-list endpoint
The canonical continuum plugin handler, the Android `BookmarksModal.vue`, booklore-ng's per-item route, and the official ABS server all return the full item bookmark list from the mutating endpoints. A separate `GET /me/item/{id}/bookmark` would be dead code.
## 5. Data model
### 5.1 Migration 148_abs_bookmarks
`migrations/148_abs_bookmarks.up.sql`:
```sql
CREATE TABLE abs_bookmarks (
id text PRIMARY KEY,
user_id integer NOT NULL,
profile_id uuid,
library_item_id text NOT NULL,
time_seconds double precision NOT NULL,
title text NOT NULL DEFAULT '',
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE UNIQUE INDEX abs_bookmarks_user_profile_item_time_uniq
ON abs_bookmarks (
user_id,
COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid),
library_item_id,
time_seconds
);
CREATE INDEX abs_bookmarks_user_item_idx
ON abs_bookmarks (user_id, library_item_id);
```
`migrations/148_abs_bookmarks.down.sql`:
```sql
DROP TABLE abs_bookmarks;
```
### 5.2 Schema rationale
- **`id` text (ULID), not UUID** — matches every other ABS surface ID emitted by silo (`abs_tokens.id`, `abs_playback_sessions.id`). Clients get a consistent shape; ULIDs are also lexicographically sortable by creation time, which simplifies debug output.
- **`profile_id` nullable, with COALESCE sentinel in the unique index** — silo's primary-profile convention is `NULL` profile. Postgres treats raw `NULL` as distinct for uniqueness, so the COALESCE-to-fixed-UUID trick collapses NULL to a single bucket per user, which is what makes "one bookmark per timestamp per profile" work.
- **Unique on `(user, profile, item, time)`** — encodes the "one bookmark per timestamp" invariant directly, which is what makes `Upsert(...,time,title)` the natural primitive: PATCH at an existing time updates the title; PATCH at a new time creates.
- **Secondary index on `(user_id, library_item_id)`** — covers the hot-path `List` query that runs after every mutation. Avoids a sort or full scan when a user has many bookmarks across many items.
- **`title` NOT NULL DEFAULT ''** — Android lets users create quick bookmarks without a title; empty string stores cleanly. Avoids null-guard branches in client iteration code.
### 5.3 Go model
```go
type Bookmark struct {
ID string // ULID
LibraryItemID string
Time float64 // fractional seconds
Title string
CreatedAt time.Time
UpdatedAt time.Time
}
```
Handlers convert to the wire shape (camelCase JSON keys, timestamps as `UnixMilli()`).
## 6. Storage contract
```go
type BookmarkStore interface {
// List returns all bookmarks for (user, profile, item) ordered by time
// ASC. Empty slice (never nil) when none exist.
List(ctx context.Context, userID, profileID, itemID string) ([]Bookmark, error)
// Upsert inserts a bookmark or updates the title at the exact
// (user, profile, item, time) tuple. ID is generated on insert and
// preserved on update. Returns the resulting row.
Upsert(ctx context.Context, userID, profileID, itemID string, time float64, title string) (Bookmark, error)
// Delete removes the bookmark at (user, profile, item, time). Returns
// nil when no row matched — DELETE is idempotent (a UX convenience,
// not a 404 surface).
Delete(ctx context.Context, userID, profileID, itemID string, time float64) error
}
```
**Behavior:**
- `Upsert` uses a single `INSERT ... ON CONFLICT (user_id, COALESCE(profile_id,'…'), library_item_id, time_seconds) DO UPDATE SET title = EXCLUDED.title, updated_at = now() RETURNING *` — one round-trip, no read-then-write race.
- `List` sort happens in SQL (`ORDER BY time_seconds ASC`), not in Go.
- Profile mapping: handlers pass `a.ProfileID` (string, empty = primary) straight through to the store; the store maps empty → NULL on writes and uses the same COALESCE-to-sentinel-UUID trick on reads. Mirrors the pattern already used by `abs_session_store.go` (`*string` round-trip) — pick whichever pgx approach reads cleanest in the impl, just keep the empty-as-primary convention consistent with the rest of `internal/audiobooks/`.
- Time precision: exact float64 equality on PATCH/DELETE-at-time. Plappa and the official client always echo back the exact value they received, so epsilon comparison would be complexity nothing exercises.
- `Delete` on a non-existent row returns `nil`. The caller still re-fetches the list and returns it, so double-tapped deletes produce consistent state.
## 7. Error model
| Condition | Status | Body |
|---|---|---|
| Missing/invalid bearer | 401 | handled by `bearerAuth` middleware |
| Body decode failure (POST/PATCH) | 400 | `invalid body` |
| `time` missing or NaN | 400 | `time required` |
| `itemId` not in `MediaStore` | 404 | `item not found` |
| Store upsert / mutate fails | 500 | sanitized `bookmark persist failed`; err logged via `slog.Error` |
| Store delete fails (DB error, not missing row) | 500 | sanitized `bookmark delete failed` |
| List fetch fails after a successful mutation | 200 + empty `[]` + `slog.Warn` | mutation already committed; returning empty list beats 500 |
**Cross-cutting:**
- **No leaked-existence side channel.** DELETE for a bookmark belonging to another user/profile returns 200 with the caller's list (which doesn't contain the target) — indistinguishable from DELETE for a non-existent bookmark. No 403, no enumeration vector.
- **Item validation.** Validate `itemId` via `MediaStore.GetAudiobookByID` before touching the store (same pattern as `handleItem`). Catches typos and deleted items early; avoids orphan bookmark rows whose item no longer exists.
- **Body size limit.** Request body wrapped in `io.LimitReader(r.Body, 1<<20)` — same 1 MiB cap as `handleStandaloneLogin`.
- **Socket publish never fails the request.** `h.publish(...)` is nil-safe and fire-and-forget; if the Publisher is unwired the response still completes.
## 8. Testing
### 8.1 Unit tests — `bookmarks_handler_test.go`
In-memory fake `BookmarkStore` (mirrors `memTokenStore` / `fakePlaybackSessionStore` pattern already used in this package).
| Test | Asserts |
|---|---|
| `Create_NewBookmark_ReturnsListContainingIt` | POST → 200, response is `[{id, time, title, …}]`, list length 1, fields populated |
| `Create_TwoAtDifferentTimes_ListOrderedByTime` | POST × 2 (times 100 and 50), list returned `[{time:50},{time:100}]` |
| `Upsert_SameTime_UpdatesTitleNoDuplicate` | POST then PATCH at same time with new title → list length 1, title updated, id preserved |
| `Delete_ExistingBookmark_RemovedFromList` | POST then DELETE same time → 200, list empty |
| `Delete_NonExistentTime_IdempotentReturnsEmptyList` | DELETE without prior POST → 200, empty list |
| `Delete_OtherUserBookmark_NoOpAndNoExistenceLeak` | seed bookmark for user B, user A DELETEs same item+time → 200, user B's bookmark untouched |
| `ProfileIsolation_BookmarksScopedPerProfile` | (user, profile A) POST → (user, profile B) List returns empty |
| `MissingItem_404` | POST against unknown itemId → 404 |
| `InvalidBody_400` | POST with malformed JSON → 400, no DB write |
| `MissingTime_400` | POST with body `{title:"x"}` (no time) → 400 |
| `SocketEvent_FiredOnCreate` | mock `EventPublisher` captures the publish; assert userID + `user_updated` + `reason:"bookmark_created"` |
| `SocketEvent_FiredOnUpdate` | same for `bookmark_updated` |
| `SocketEvent_FiredOnDelete` | same for `bookmark_deleted` |
### 8.2 Wire-shape test — `bookmarks_envelope_test.go`
One marshal test that takes a `Bookmark` (including the empty-title case) and asserts every required JSON key is present and uses the camelCase spelling: `id`, `libraryItemId`, `time`, `title`, `createdAt`, `updatedAt`. Mirrors the existing `TestSiloItemToMetadata_JSONKeysAlwaysPresent` and `TestLoginEnvelope_HasRequiredKeys` patterns.
### 8.3 Live integration smoke (post-deploy)
```
TOKEN=$(curl -s -X POST -H 'Content-Type: application/json' -H 'x-return-tokens: true' \
-d '{"username":"<u>","password":"<p>"}' http://127.0.0.1:13378/login \
| python3 -c "import sys,json;print(json.load(sys.stdin)['accessToken'])")
ITEM=$(curl -s -H "Authorization: Bearer $TOKEN" \
http://127.0.0.1:13378/api/libraries/<lid>/items?limit=1 \
| python3 -c "import sys,json;print(json.load(sys.stdin)['results'][0]['id'])")
curl -s -X POST -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
-d '{"title":"smoke","time":42.5}' \
http://127.0.0.1:13378/api/me/item/$ITEM/bookmark | python3 -m json.tool
curl -s -X PATCH -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
-d '{"title":"updated","time":42.5}' \
http://127.0.0.1:13378/api/me/item/$ITEM/bookmark | python3 -m json.tool
curl -s -X DELETE -H "Authorization: Bearer $TOKEN" \
http://127.0.0.1:13378/api/me/item/$ITEM/bookmark/42.5 | python3 -m json.tool
```
Expected: each call returns the updated list; the final DELETE returns `[]`.
### 8.4 Explicitly out-of-scope
- No real-Postgres integration test for the SQL. The `ON CONFLICT` clause and unique-index COALESCE behavior are small enough that the in-memory fake covers the semantics; CI does not currently run a Postgres for the ABS subdomain.
- No load test. Bookmark mutations are user-driven and rare (handful per session).
## 9. Risks & open questions
- **Time-precision collision risk.** If a future client rounds `time` differently between POST and DELETE (e.g. POST sends `42.500001`, DELETE sends `42.5`), the DELETE would silently no-op. Acceptable for the current client set (Android and official iOS echo exactly), but worth a comment on the `Delete` method so future-us doesn't burn time chasing a ghost.
- **No migration rollback test.** The `.down.sql` is one `DROP TABLE` — low risk — but the migration suite doesn't currently exercise rolls. Same posture as recent migrations; not a blocker.
- **Bookmark fan-out on socket events.** `Handler.publish(userID, ...)` targets the user room. If the user has many connected sockets, the publish fans out per-socket. Volume here is trivial (manual user action). No backpressure concern at Phase 1 scale.
## 10. Out-of-scope follow-ups
These are intentionally deferred — they belong to later Phase 1 sub-projects or Phase 2:
- Aggregate `GET /api/me/bookmarks` (caller's full bookmark inventory). Adds value only when a "Bookmarks" tab lands in a client that needs it.
- Hydrating `user.bookmarks` in the `/login` envelope. Currently emits `[]` placeholder; populating costs a join on every login and only matters to clients that read from there at startup.
- Embedding `bookmarks` on item-detail (`GET /api/items/{id}`). The Android player fetches its bookmarks via the per-item bookmark endpoint on modal-open, not from item-detail.
## 11. References
- Spec parent: `docs/superpowers/specs/2026-05-26-abs-implementation-fix-design.md`.
- Wire-shape reference (canonical, working against real ABS Android): the continuum-plugin-audiobooks `bookmarks_handler.go` + the `r.Post/Patch/Delete` mounts in its `handler.go`. Diff against silo before flagging response-shape concerns.
- Client wire usage: `audiobookshelf-app/components/modals/BookmarksModal.vue` (the only place the official client builds bookmark requests).
- Booklore-ng has its own `me/bookmarks` aggregate endpoint that the official client does not use; ignored here.
docs/superpowers/specs/2026-05-26-abs-collections-playlists-design.md
+523
View File
@@ -0,0 +1,523 @@
# ABS Collections + Playlists — Phase 1 Sub-Project 2
**Status:** Approved 2026-05-26. Ready for implementation plan.
**Scope:** Second sub-project of Phase 1 (per `2026-05-26-abs-implementation-fix-design.md`).
**Predecessor spec:** `docs/superpowers/specs/2026-05-26-abs-implementation-fix-design.md` §"Phase 1 — Feature surface completion" → Manual Collections + Playlists bullets.
**Sibling sub-project:** `docs/superpowers/specs/2026-05-26-abs-bookmarks-design.md` (sub-project 1; the bookmarks data model, in-memory test fakes, store wiring, and error-handling conventions established there are reused here without re-justification).
Commands in this document assume the repository root is the cwd.
## 1. Goal
Land manual user collections and ordered playlists on the silo audiobook surface so the official Audiobookshelf Android, iOS, and Plappa clients can create, browse, mutate, and share named groupings of audiobooks. "Manual collection" means a user-curated unordered set of audiobooks with a name + description (think: "Favorites", "To Read"). "Playlist" means an ordered queue with a cover image (think: "Wind-down listening"). Both are owned by a profile, optionally public to other users on the same silo instance.
## 2. Non-goals
- **No podcast episode hydration.** Playlist items accept and echo `episodeId`, but only audiobook items are looked up in `MediaStore`. A future sub-project will wire `internal/audiobooks/podcastfeed` hydration.
- **No reorder API.** Clients simulate reorder via remove+add (the re-added item lands at the end).
- **No smart collections.** Those are sub-project 3 (separate `query_def` storage + DSL evaluator).
- **No silo-web UI.** The web admin doesn't ship a collections/playlists view yet; ABS mobile clients render these.
- **No socket-server overhaul.** Playlists publish three existing-pattern events; collections publish none. Full socket parity is Phase 2 of the parent spec.
## 3. Architecture
Two REST surfaces sharing a uniform shape, mounted under `/abs/api/*` and `/api/*` inside the existing `bearerAuth` group.
- **HTTP layer:**
- `internal/audiobooks/abs/collections_handler.go` — 7 routes (list, create, get, update, delete, add item, remove item).
- `internal/audiobooks/abs/playlists_handler.go` — 9 routes (list, create, get, update, delete, add single item, batch add, batch remove, remove single item; remove-episode variant uses two URL params).
- **Storage layer:**
- New interfaces in `internal/audiobooks/abs/collections.go` and `playlists.go` (one file each, parallel to `bookmarks.go`):
- `CollectionStore` — `ListUserCollections / GetCollection / CreateCollection / UpdateCollection / DeleteCollection / ListCollectionItems / AddCollectionItem / RemoveCollectionItem`.
- `PlaylistStore` — `ListUserPlaylists / GetPlaylist / CreatePlaylist / UpdatePlaylist / DeletePlaylist / ListPlaylistItems / AddPlaylistItem / RemovePlaylistItem`.
- Concrete pgx impls in `internal/audiobooks/abs_collection_store.go` and `abs_playlist_store.go` (parallel to `abs_bookmark_store.go`).
- **Migrations** (paired up/down per `CLAUDE.md`):
- `149_abs_user_collections`, `150_abs_collection_items`, `151_abs_playlists`, `152_abs_playlist_items`.
- **Socket events:** Playlists publish `playlist_added` / `playlist_updated` / `playlist_removed` (continuum-canonical event names). Collections publish nothing. All publishes go through the existing nil-safe `Handler.publish(...)` wrapper.
- **Service wiring:** Both new stores wired in `BuildABSHandler` (mirrors the BookmarkStore wiring landed in sub-project 1).
## 4. Endpoint surface
All routes mounted under both `/abs/api/*` and `/api/*` inside `bearerAuth`. Success status code is **200 OK** in every row of the tables below, except where the `Returns` column explicitly says **204 No Content** (DELETE collection / DELETE playlist themselves; the item-mutation DELETEs return 200 with the parent's full-shape body, matching continuum).
### 4.1 Collections (7 routes)
| Verb | Path | Body | Returns |
|---|---|---|---|
| GET | `/collections` | — | `{"collections": [Collection list-shape]}` |
| POST | `/collections` | `{name, description, isPublic?}` | Collection full-shape |
| GET | `/collections/{id}` | — | Collection full-shape when caller is owner, OR when `is_public=true`; otherwise 404 |
| PATCH | `/collections/{id}` | `{name?, description?, isPublic?}` | Collection full-shape |
| DELETE | `/collections/{id}` | — | 204 No Content |
| POST | `/collections/{id}/book/{bookId}` | — | Collection full-shape (with updated `books[]`) |
| DELETE | `/collections/{id}/book/{bookId}` | — | Collection full-shape |
### 4.2 Playlists (9 routes)
| Verb | Path | Body | Returns |
|---|---|---|---|
| GET | `/playlists` | — | `{"playlists": [Playlist list-shape]}` |
| POST | `/playlists` | `{name, description, cover_item?, isPublic?}` | Playlist full-shape |
| GET | `/playlists/{id}` | — | Playlist full-shape when caller is owner, OR when `is_public=true`; otherwise 404 |
| PATCH | `/playlists/{id}` | `{name?, description?, cover_item?, isPublic?}` | Playlist full-shape |
| DELETE | `/playlists/{id}` | — | 204 No Content |
| POST | `/playlists/{id}/item` | `{libraryItemId, episodeId?}` | Playlist full-shape |
| POST | `/playlists/{id}/batch/add` | `{items: [{libraryItemId, episodeId?}]}` | Playlist full-shape |
| POST | `/playlists/{id}/batch/remove` | `{items: [{libraryItemId, episodeId?}]}` | Playlist full-shape |
| DELETE | `/playlists/{id}/item/{libraryItemId}` | — | Playlist full-shape |
| DELETE | `/playlists/{id}/item/{libraryItemId}/{episodeId}` | — | Playlist full-shape |
### 4.3 Wire shape — Collection
**List shape (no `books[]`):**
```json
{
"id": "01HXXX",
"userId": "1",
"name": "Favorites",
"description": "My top picks",
"isPublic": false,
"lastUpdate": 1779786284823,
"createdAt": 1779786284823
}
```
**Full shape (with `books[]`):**
```json
{
"id": "01HXXX",
"userId": "1",
"name": "Favorites",
"description": "My top picks",
"isPublic": false,
"lastUpdate": 1779786284823,
"createdAt": 1779786284823,
"books": [
{
"id": "126887...",
"libraryId": "9",
"media": { "metadata": { "title": "Book Title", "authors": [...] } }
}
]
}
```
All seven top-level keys always present (`description` is `""` when unset, never omitted — fixes the continuum-reference bug where description was always emitted as empty regardless of stored value). `books[]` is always an array (possibly empty) on the full shape; omitted on the list shape.
### 4.4 Wire shape — Playlist
**List shape (no `items[]`):**
```json
{
"id": "01HXXX",
"userId": "1",
"name": "My Queue",
"description": "",
"isPublic": false,
"coverPath": "126887...",
"createdAt": 1779786284823,
"lastUpdate": 1779786284823
}
```
**Full shape (with `items[]`):**
```json
{
"id": "01HXXX",
"userId": "1",
"name": "My Queue",
"description": "",
"isPublic": false,
"coverPath": "126887...",
"createdAt": 1779786284823,
"lastUpdate": 1779786284823,
"items": [
{ "libraryItemId": "126887...", "position": 0, "title": "Book Title", "libraryId": "9" },
{ "libraryItemId": "podcast-x", "episodeId": "ep-1", "position": 1 }
]
}
```
`coverPath` is emitted when set; omitted when unset. `description` always present. Items always sorted by `position` ASC. Entries with `episodeId` set are emitted with the field; audiobook entries omit it. Audiobook items hydrate `title` + `libraryId` via `MediaStore.GetAudiobookByID`; episode items emit the bare reference.
### 4.5 Hydration semantics
- `MediaStore.GetAudiobookByID(libraryItemId)` is called per book on the **full** shape only (the list shape skips it to keep response sizes small).
- On miss (item deleted between collection/playlist mutation and list-fetch), the entry degrades to `{id, libraryId}` only — clients render a placeholder.
- `libraryId` resolution: same `resolveDefaultLibrary` helper from `items_handler.go` (returns numeric library ID as decimal string).
### 4.6 Socket events
| Event | When | Payload |
|---|---|---|
| `playlist_added` | After POST `/playlists` | `{id, name}` |
| `playlist_updated` | After PATCH or any item mutation (single or batch) | `{id}` |
| `playlist_removed` | After DELETE `/playlists/{id}` | `{id}` |
Collections fire no socket events (matches continuum; Phase 2 of the parent spec covers full event parity).
## 5. Data model
### 5.1 Migration 149 — `abs_user_collections`
`migrations/149_abs_user_collections.up.sql`:
```sql
CREATE TABLE IF NOT EXISTS public.abs_user_collections (
id text PRIMARY KEY,
user_id integer NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
profile_id uuid,
name text NOT NULL,
description text NOT NULL DEFAULT '',
is_public boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS abs_user_collections_user_profile_idx
ON public.abs_user_collections (
user_id,
COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid)
);
```
`migrations/149_abs_user_collections.down.sql`:
```sql
DROP INDEX IF EXISTS public.abs_user_collections_user_profile_idx;
DROP TABLE IF EXISTS public.abs_user_collections;
```
### 5.2 Migration 150 — `abs_collection_items`
`migrations/150_abs_collection_items.up.sql`:
```sql
CREATE TABLE IF NOT EXISTS public.abs_collection_items (
collection_id text NOT NULL REFERENCES public.abs_user_collections(id) ON DELETE CASCADE,
library_item_id text NOT NULL REFERENCES public.media_items(content_id) ON DELETE CASCADE,
added_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (collection_id, library_item_id)
);
CREATE INDEX IF NOT EXISTS abs_collection_items_library_item_idx
ON public.abs_collection_items (library_item_id);
```
`migrations/150_abs_collection_items.down.sql`:
```sql
DROP INDEX IF EXISTS public.abs_collection_items_library_item_idx;
DROP TABLE IF EXISTS public.abs_collection_items;
```
### 5.3 Migration 151 — `abs_playlists`
`migrations/151_abs_playlists.up.sql`:
```sql
CREATE TABLE IF NOT EXISTS public.abs_playlists (
id text PRIMARY KEY,
user_id integer NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
profile_id uuid,
name text NOT NULL,
description text NOT NULL DEFAULT '',
cover_item text REFERENCES public.media_items(content_id) ON DELETE SET NULL,
is_public boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS abs_playlists_user_profile_idx
ON public.abs_playlists (
user_id,
COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid)
);
```
`migrations/151_abs_playlists.down.sql`:
```sql
DROP INDEX IF EXISTS public.abs_playlists_user_profile_idx;
DROP TABLE IF EXISTS public.abs_playlists;
```
### 5.4 Migration 152 — `abs_playlist_items`
`migrations/152_abs_playlist_items.up.sql`:
```sql
CREATE TABLE IF NOT EXISTS public.abs_playlist_items (
playlist_id text NOT NULL REFERENCES public.abs_playlists(id) ON DELETE CASCADE,
library_item_id text NOT NULL,
episode_id text NOT NULL DEFAULT '',
position integer NOT NULL,
added_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (playlist_id, library_item_id, episode_id)
);
CREATE INDEX IF NOT EXISTS abs_playlist_items_playlist_position_idx
ON public.abs_playlist_items (playlist_id, position);
```
`migrations/152_abs_playlist_items.down.sql`:
```sql
DROP INDEX IF EXISTS public.abs_playlist_items_playlist_position_idx;
DROP TABLE IF EXISTS public.abs_playlist_items;
```
### 5.5 Schema rationale
- **`is_public boolean`** — cross-user-public read semantics. `false` default. List endpoints never expose other users' rows; GET-by-id allows non-owner reads only when `is_public = true`.
- **Collection items: FK + CASCADE on `library_item_id`** — when a book is deleted from the library, drop it from all collections (clean orphan-free state). Same pattern as migration 143's `abs_playback_sessions.content_id`.
- **Collection items: composite PK on `(collection_id, library_item_id)`** — enforces "a book appears at most once in a collection" without a synthetic ID.
- **Playlist items: `library_item_id` NOT FK'd** — collections enforce one-shot membership; playlists are ordered queues that may legitimately reference items the user hasn't bookmarked. Decoupling lets a future migration FK it when episode support lands properly. Today: handler validates via MediaStore (same pattern as bookmarks).
- **Playlist items: `episode_id text NOT NULL DEFAULT ''`** — empty string for audiobook items, populated for podcast episodes. Empty-string-default lets `(playlist_id, library_item_id, episode_id)` be a clean unique key without COALESCE.
- **Playlist items: no synthetic ID** — `(playlist_id, library_item_id, episode_id)` is naturally unique; `position` is sort hint (gaps allowed when items are removed).
- **`cover_item REFERENCES media_items(content_id) ON DELETE SET NULL`** — playlist survives cover-item deletion; cover gracefully becomes nothing.
### 5.6 Go models
```go
type Collection struct {
ID string
UserID string
ProfileID string
Name string
Description string
IsPublic bool
CreatedAt time.Time
UpdatedAt time.Time
}
type CollectionItem struct {
CollectionID string
LibraryItemID string
AddedAt time.Time
}
type Playlist struct {
ID string
UserID string
ProfileID string
Name string
Description string
CoverItem string // empty when unset
IsPublic bool
CreatedAt time.Time
UpdatedAt time.Time
}
type PlaylistItem struct {
PlaylistID string
LibraryItemID string
EpisodeID string // empty for audiobook items
Position int
AddedAt time.Time
}
```
Handlers convert these to the wire shape (camelCase JSON keys, timestamps as `UnixMilli()`).
## 6. Storage contract
### 6.1 `CollectionStore` (in `internal/audiobooks/abs/collections.go`)
```go
type CollectionStore interface {
ListUserCollections(ctx context.Context, userID, profileID string) ([]Collection, error)
GetCollection(ctx context.Context, id string) (Collection, error)
CreateCollection(ctx context.Context, c Collection) error
UpdateCollection(ctx context.Context, c Collection) error
DeleteCollection(ctx context.Context, id string) error
ListCollectionItems(ctx context.Context, collectionID string) ([]CollectionItem, error)
AddCollectionItem(ctx context.Context, collectionID, libraryItemID string) error
RemoveCollectionItem(ctx context.Context, collectionID, libraryItemID string) error
}
```
### 6.2 `PlaylistStore` (in `internal/audiobooks/abs/playlists.go`)
```go
type PlaylistStore interface {
ListUserPlaylists(ctx context.Context, userID, profileID string) ([]Playlist, error)
GetPlaylist(ctx context.Context, id string) (Playlist, error)
CreatePlaylist(ctx context.Context, p Playlist) error
UpdatePlaylist(ctx context.Context, p Playlist) error
DeletePlaylist(ctx context.Context, id string) error
ListPlaylistItems(ctx context.Context, playlistID string) ([]PlaylistItem, error)
AddPlaylistItem(ctx context.Context, playlistID, libraryItemID, episodeID string) error
RemovePlaylistItem(ctx context.Context, playlistID, libraryItemID, episodeID string) error
}
```
### 6.3 Behavior
- **List ordering:** Collections list ordered by `created_at DESC`. Playlists list ordered by `created_at DESC`. Playlist items ordered by `position ASC`. Collection items ordered by `added_at ASC`. All sorting happens in SQL, not in Go.
- **Empty results:** All List methods return an empty slice (never nil) when no rows match.
- **`GetCollection` / `GetPlaylist`:** Return `ErrNotFound` when absent. NO owner check — caller authorizes via the response's `UserID` and `IsPublic`.
- **`AddPlaylistItem` position assignment:** Single SQL statement computing `position = COALESCE(MAX(position), 0) + 1 WHERE playlist_id = $1` inside the INSERT. No read-before-write race.
- **`AddCollectionItem` / `AddPlaylistItem` idempotency:** `INSERT ... ON CONFLICT (...) DO NOTHING`. Re-adding an existing tuple is a silent no-op (no error, no row mutation, no `updated_at` bump beyond what the handler does separately).
- **`RemoveCollectionItem` / `RemovePlaylistItem` idempotency:** Return `nil` on no-match.
- **Profile mapping:** Handlers pass `a.ProfileID` straight through (empty = primary). Store uses `profileArg` helper (returns `nil` for empty, the string otherwise). `COALESCE(profile_id, '00000000-...'::uuid)` in WHERE clauses on both column and bind value, mirroring the BookmarkStore convention.
- **`updated_at` bump on item mutations:** Both `Add*Item` and `Remove*Item` execute the item mutation and an `UPDATE abs_user_collections SET updated_at = now() WHERE id = $1` (or `abs_playlists`) in the same transaction. Wire field `lastUpdate` reflects this.
- **Cascade cleanup:** Deleting a collection drops all its `abs_collection_items` via FK CASCADE. Deleting a playlist drops all its `abs_playlist_items`. Deleting a `media_items` row drops it from all `abs_collection_items` (via FK CASCADE); playlists surface stale references on the next List, where the handler degrades to bare-id entries.
### 6.4 Cross-user public read
Handler pattern after `Get*`:
```go
c, err := store.GetCollection(ctx, id)
if errors.Is(err, ErrNotFound) || (c.UserID != a.UserID && !c.IsPublic) {
http.Error(w, "collection not found", http.StatusNotFound)
return
}
```
Same pattern for playlists. The condition collapses unknown-id and not-authorized into one branch so existence-leak vectors don't open up.
## 7. Error model
| Condition | Status | Body |
|---|---|---|
| Missing/invalid bearer | 401 | handled by `bearerAuth` middleware |
| Body decode failure (POST/PATCH) | 400 | `invalid body` |
| Required field missing (POST: `name`; POST item: `libraryItemId`) | 400 | `<field> required` |
| Unknown collection/playlist on GET (owner or not) | 404 | `collection not found` / `playlist not found` |
| Non-owner GET on private collection/playlist | 404 | same body — no existence leak |
| Non-owner PATCH/DELETE/item-mutation | 404 | same body |
| `library_item_id` not in MediaStore (audiobook add-item) | 404 | `item not found` |
| Store insert/update/delete fails | 500 | sanitized `<collection\|playlist> persist failed`; err logged via `slog.Error` |
| List fetch fails after a successful mutation | 200 + best-effort response (in-memory state) + `slog.Warn` | mutation already committed; falling back beats 500 |
### 7.1 Cross-cutting
- **Anti-enumeration.** All non-owner-or-private paths funnel to the same `<type> not found` 404 (never 403, never "private collection"). Indistinguishable from real not-found.
- **Item validation on add.** `handleAddCollectionBook` and `handleAddPlaylistItem` call `MediaStore.GetAudiobookByID(libraryItemID)` before touching the store. Catches typos and orphan refs early. **Exception:** `handleAddPlaylistItem` with non-empty `episodeId` SKIPS item validation (audiobook-only hydration scope; podcast episodes are stored opaque-id-style).
- **Body size limit.** All POST/PATCH wrap `r.Body` in `io.LimitReader(r.Body, 1<<20)` before `json.NewDecoder`. Same as bookmarks.
- **Batch endpoints.** `POST /playlists/{id}/batch/add` and `/batch/remove` tolerate per-item failures silently (matching continuum's `_ = h.store.Add...`). Total failure (e.g., body decode error) is still a 400. Item validation on the batch path: each item validated individually; failed validations skipped with `slog.Debug` (no 404 — the operation as a whole succeeds with the remaining items).
- **Socket event on batch mutation.** A batch add/remove fires exactly one `playlist_updated` event, regardless of how many items succeeded or failed. Clients re-render from the response.
- **Socket publish never fails the request.** `h.publish(...)` is nil-safe and fire-and-forget; if the Publisher is unwired the response still completes.
## 8. Testing
### 8.1 Unit tests — `collections_handler_test.go`
In-memory fake `memCollectionStore` (parallel to `memBookmarkStore`). Reuses `stubMediaStore` and `recordingPublisher` from `bookmarks_handler_test.go`. The `dispatchBookmark` helper is generalized in-place (or a sibling `dispatchABS` is added) so the same wiring (URL params + `ctxAuth` injection) drives both new surfaces.
| Test | Asserts |
|---|---|
| `Collection_Create_ReturnsFullShape` | POST `{name:"x"}` → 200, response carries all 7 top-level keys + empty `books[]`, ID is a ULID |
| `Collection_Create_NameRequired_400` | POST `{description:"only"}` → 400 |
| `Collection_List_ReturnsWrappedEnvelope` | GET → 200, body is `{"collections":[…]}`, list-shape omits `books` |
| `Collection_List_DoesNotLeakOtherUsers` | User 1 creates; User 2 GETs → empty list |
| `Collection_Get_Owner_ReturnsFullShape` | POST → owner GET → 200 + `books[]` |
| `Collection_Get_NonOwner_Public_OK` | User 1 creates with `isPublic:true`; User 2 GET → 200 |
| `Collection_Get_NonOwner_Private_404` | User 1 creates (default private); User 2 GET → 404 |
| `Collection_Patch_OwnerUpdatesNameAndDescription` | POST → PATCH `{name:"y", description:"d"}` → 200, fields updated, `lastUpdate` advanced |
| `Collection_Patch_NonOwner_404` | User 2 PATCH on User 1's collection → 404 (no existence leak) |
| `Collection_Delete_OwnerRemovesItAndItems` | POST + add book → DELETE → 204; subsequent GET → 404; items table empty |
| `Collection_Delete_NonOwner_404` | User 2 DELETE → 404, User 1's collection still present |
| `Collection_AddBook_Owner_HydratesInResponse` | POST + add book → 200 + `books[]` contains the entry with `media.metadata.title` |
| `Collection_AddBook_Idempotent` | Add same book twice → 200 both times, `books[]` length stays at 1 |
| `Collection_AddBook_UnknownItem_404` | Add a non-existent item → 404 `item not found` |
| `Collection_RemoveBook_Idempotent` | Remove book not in collection → 200, `books[]` unchanged |
| `Collection_ProfileIsolation` | Profile A creates; Profile B's list returns empty |
| `Collection_Envelope_HasRequiredKeys` | Marshal-test: 7 top-level keys present even when description and books are empty |
### 8.2 Unit tests — `playlists_handler_test.go`
In-memory fake `memPlaylistStore`.
| Test | Asserts |
|---|---|
| `Playlist_Create_ReturnsFullShape` | POST `{name:"x", description:"d", isPublic:true}` → 200, all 8 keys present, ID is ULID |
| `Playlist_Create_NameRequired_400` | POST `{description:"only"}` → 400 |
| `Playlist_Create_FiresPlaylistAddedEvent` | POST → `recordingPublisher` captures `playlist_added` with `{id, name}` |
| `Playlist_List_WrappedEnvelope` | GET → 200, body `{"playlists":[…]}`, list-shape omits `items` |
| `Playlist_AddItem_AppendsAtNextPosition` | POST + add 3 items → response `items[]` positions are 1,2,3 in insertion order |
| `Playlist_AddItem_Idempotent` | Add same `(libraryItemId, episodeId)` twice → list length 1 |
| `Playlist_AddItem_AudiobookHydrates` | Add audiobook item → response item has `title` |
| `Playlist_AddItem_Episode_AcceptsAndEchoes` | Add `{libraryItemId, episodeId:"ep-1"}` → response item has `episodeId`, no `title` (un-hydrated by design) |
| `Playlist_AddItem_UnknownAudiobook_404` | Add non-existent audiobook → 404; episode adds skip validation, so episode-only adds succeed |
| `Playlist_BatchAdd_TolerantOfPartialFailures` | Batch of `[valid, invalid, valid]` → 200, response has 2 items, no 404 |
| `Playlist_BatchAdd_FiresOneUpdatedEvent` | Batch add → single `playlist_updated` event (not one per item) |
| `Playlist_BatchRemove` | Seed 3 items, batch-remove 2 → response items[] length 1 |
| `Playlist_RemoveItem_Single` | DELETE /item/{libraryItemId} → 200, item gone |
| `Playlist_RemoveItem_WithEpisode` | DELETE /item/{libraryItemId}/{episodeId} → 200, episode-keyed item removed; audiobook-keyed item with same libraryItemId still present |
| `Playlist_Patch_UpdatesCover` | PATCH `{cover_item:"id"}` → 200, `coverPath` updated; `playlist_updated` event fires |
| `Playlist_Delete_FiresPlaylistRemovedEvent` | DELETE → 204 + `playlist_removed` event |
| `Playlist_Get_NonOwner_Public_OK` | User 1 creates `isPublic:true`; User 2 GET → 200 |
| `Playlist_Get_NonOwner_Private_404` | Private playlist; non-owner GET → 404 |
| `Playlist_NonOwner_Mutation_404` | User 2 add-item / remove-item / PATCH / DELETE → 404 each, User 1's playlist intact |
| `Playlist_ProfileIsolation` | Profile A creates; Profile B's list returns empty |
| `Playlist_Envelope_HasRequiredKeys` | Marshal-test: 8 top-level keys present; `coverPath` omitted when empty |
### 8.3 Shared envelope tests
One marshal test per surface (`collections_envelope_test.go`, `playlists_envelope_test.go`) asserting the wire shape end-to-end. Parallel to `bookmarks_envelope_test.go`.
### 8.4 Live integration smoke (post-deploy, operator)
```bash
TOKEN=$(curl ... /login | jq -r .accessToken)
ITEM=$(curl ... /api/libraries/9/items?limit=1 | jq -r .results[0].id)
# Collections
COLL=$(curl -X POST -d '{"name":"smoke"}' ... /api/collections | jq -r .id)
curl -X POST ... /api/collections/$COLL/book/$ITEM
curl ... /api/collections/$COLL # books[] contains $ITEM
curl -X DELETE ... /api/collections/$COLL/book/$ITEM
curl -X DELETE ... /api/collections/$COLL # 204
# Playlists
PL=$(curl -X POST -d '{"name":"queue"}' ... /api/playlists | jq -r .id)
curl -X POST -d "{\"libraryItemId\":\"$ITEM\"}" ... /api/playlists/$PL/item
curl ... /api/playlists/$PL # items[] contains $ITEM at position 1
curl -X DELETE ... /api/playlists/$PL/item/$ITEM
curl -X DELETE ... /api/playlists/$PL # 204
```
Expected: each call returns the wire shape from §4; `playlist_added` / `_updated` / `_removed` events appear on the user's Socket.io channel.
### 8.5 Explicitly out of scope
- **No real-Postgres SQL test.** Same posture as bookmarks (§8.4 of that spec): in-memory fakes cover the semantics; CI does not run Postgres for the ABS subdomain.
- **No episode hydration.** Episode IDs accepted and echoed but not resolved to podcast metadata. A future podcast-playlist sub-project will plug in `podcastfeed` hydration.
- **No reorder API.** Future follow-up.
- **No two-user smoke.** Cross-user-public visibility is unit-tested but the operator smoke uses one user.
## 9. Risks & open questions
- **Position gaps after remove.** Removing the middle item from a playlist leaves a `position` gap (1, 3, 4 instead of 1, 2, 3). Clients sort by position; gaps are harmless. If a future reorder API lands, it'll compact positions in a single statement.
- **Cross-user public + profile scoping interaction.** A public collection owned by `(user A, profile primary)` is visible to user B regardless of B's active profile. The collection's `userId` field discloses A's user identity. Acceptable for v1; if A wanted to hide who owns the collection, that's a separate feature.
- **No migration rollback test.** Down migrations are `DROP INDEX + DROP TABLE` — low risk, same posture as bookmarks. Not a blocker.
- **`cover_item` references a `media_items.content_id` but the playlist owner may not have access to that library** — silo's library visibility isn't enforced at the schema level. The cover image hydrator (when added) will need to check visibility. For now, the field round-trips opaquely.
- **Concurrent `AddPlaylistItem` position collision.** Two parallel appends compute `MAX(position)+1` concurrently and may both produce the same value. The UNIQUE constraint on `(playlist_id, library_item_id, episode_id)` prevents collisions for the SAME item, but distinct items could land at the same position. Acceptable — clients tolerate equal positions and break ties by insertion order. A SERIALIZABLE transaction would close the race; not warranted for a low-traffic UX surface.
## 10. Out-of-scope follow-ups
These are intentionally deferred — they belong to later Phase 1 sub-projects or Phase 2:
- **Smart collections** — sub-project 3 (`abs_smart_collections` + `query_def` JSONB + DSL evaluator).
- **RSS feeds** — sub-project 4 (`abs_rss_feeds`).
- **Listening stats** — sub-project 4 (aggregations on `abs_playback_sessions`).
- **Author / series detail endpoints** — small sub-project, separate.
- **Continue-listening toggles** — small sub-project, separate.
- **Reorder API for playlists** — compaction + drag-reorder UX. Probably warrants its own design pass.
- **Cover-image hydration** — currently the wire emits `coverPath: <content_id>` opaquely. A future enhancement resolves it through `DetailService.PresignURL` to a fully-qualified cover URL.
- **Episode hydration in playlists** — when the podcast-playlist sub-project lands.
- **Collection socket events** — Phase 2 of the parent spec.
## 11. References
- Spec parent: `docs/superpowers/specs/2026-05-26-abs-implementation-fix-design.md`.
- Sibling sub-project spec: `docs/superpowers/specs/2026-05-26-abs-bookmarks-design.md`.
- Wire-shape reference: `continuum-plugin-audiobooks/internal/abs/collections_handler.go` and `playlists_handler.go` (canonical, diff against silo's adaptation before flagging response-shape concerns).
- Client wire usage: `audiobookshelf-app/components/...` Collections and Playlists modal/page components.
docs/superpowers/specs/2026-05-26-abs-implementation-fix-design.md
+384
View File
@@ -0,0 +1,384 @@
# ABS Implementation Fix — Design
**Status:** Approved (brainstorming complete, awaiting writing-plans handoff)
**Date:** 2026-05-26
**Scope:** Bring silo-server's Audiobookshelf-compatible API to full parity with the canonical `continuum-plugin-audiobooks` implementation so that official ABS iOS, Android, and 3rd-party (Plappa, AudioBookShelfFully) clients work end-to-end against silo.
**Out of scope:** silo's native audiobook surface (`/api/v1/audiobooks/*`), silo-android/silo-apple/silo-plugin-sdk repos, Continuum's plugin-host RPC layer, the "standalone listener" port (silo has no separate process), cover transcoding pipelines, transcribed-audio search.
Commands assume the repository root is the cwd.
---
## 1. Problem
silo's ABS-compat layer at `internal/audiobooks/abs/` exposes ~20% of the surface that the canonical Continuum plugin implements (`continuum-plugin-audiobooks` reference). Real ABS mobile clients cannot complete the basic flow today — login is reported broken, and even when it succeeds many subsequent calls (filterdata, resume position, author/series IDs) return data shapes that break the client. Bookmarks, collections, playlists, smart collections, RSS feeds, author/series detail, and listening stats are entirely absent. Socket.io publishes only 3 of the ~30 events real clients subscribe to.
The user's directive: every feature of ABS clients must function. Source of truth for behavior is the Continuum plugin; the booklore-ng reference docs (`BOOKLORE_ABS_IMPLEMENTATION_ISSUES.md`, `src/lib/socket/events.ts`) document specific response-shape bugs and the canonical socket event list.
## 2. Goals & Non-goals
### Goals
- Official ABS iOS app, ABS Android app, and 3rd-party clients (Plappa, AudioBookShelfFully) work end-to-end: add server → login → browse libraries → play → progress sync → bookmark → collection → playlist.
- Token lifecycle is complete: login, refresh, logout, revocation, multi-device tracking.
- Socket.io publishes the full event surface real clients subscribe to.
- New data lives in ABS-scoped tables that don't entangle silo's existing collection system.
### Non-goals
- silo's native audiobook UI changes.
- silo-android / silo-apple client changes.
- Importing real audiobook content via ABS-protocol requests (silo has its own `internal/requests/` system that is not being bridged here).
- Watch-together, podcast download queues, server-side backup event streams.
## 3. Architecture & Strategy
**Source of truth:** `continuum-plugin-audiobooks` (canonical). Port file-by-file, adapt to silo's data model (catalog repos, `auth.Service`, profile model).
**Topology:** silo is monolithic; the ABS layer reads/writes the local DB directly. The Continuum plugin's `HostClient → backend plugin RPC` layer has no analog and is dropped.
**Package layout:** keep silo's existing `internal/audiobooks/{abs,abssocket,podcastfeed}` — it already mirrors the Continuum plugin's structure.
**Listener:** continue using the dedicated `:13378` ABS-compat HTTP server (configured in `internal/config/db_loader.go` as `audiobookshelf_compat.listen`). Routes mounted on a fresh chi router; no SPA fallback collisions.
**Data isolation:** new ABS surface uses dedicated tables (`abs_bookmarks`, `abs_user_collections`, `abs_collection_items`, `abs_playlists`, `abs_playlist_items`, `abs_smart_collections`, `abs_rss_feeds`). silo's existing `library_collections` / `user_personal_collections` / `user_smart_collections` / collection groups are untouched. Future bridging is possible but out of scope here.
## 4. Phasing
Four phases, each independently mergeable and independently verifiable against a real mobile client.
### Phase 0 — Login + critical bug fixes
**Goal:** iOS/Android app can `add server → login → browse library → tap book → play` end-to-end. No bookmarks/collections/etc. yet.
**Concrete changes:**
| File:line | Change |
|---|---|
| `internal/audiobooks/abs/login.go:195-234` | Add to login envelope: `user.itemTagsAccessible: []`, `user.itemTagsSelected: []`, `user.lastSeen: <epoch ms>`, `user.createdAt: <epoch ms>`. Enrich `serverSettings` with `coverAspectRatio`, `dateFormat`, `timeFormat`, `storeCoverWithItem`, `scannerDisableWatcher`, `metadataFileFormat`, `chromecastEnabled`. Mirror Continuum's `completeLogin` (`continuum-plugin-audiobooks/internal/abs/handler.go:591-697`) verbatim. |
| `internal/audiobooks/abs/login.go:274-315` (`handleABSAuthorize`) | Return the identical envelope as `/login`, including `accessToken` and `refreshToken`. Currently omits them, which breaks resume-on-launch. |
| `internal/audiobooks/abs/handler.go:353-391` (bearerAuth) | Audit JTI lookup against `abs_session_store.go:GetTokenByJTI` to confirm the freshly-minted JTI is found. Add `slog.Debug` lines at each rejection branch so the next 401 is traceable. Confirm `?token=` query-param fallback parses correctly (iOS AVPlayer requirement). |
| `internal/audiobooks/abs/libraries_handler.go:499-549` (`siloItemToMetadata`) | Include `id` on every `authors[]` entry (use `item_people.id` UUID). Include `id` on every `series[]` entry (slugified name until a series table lands). Populate `genres` and `tags` from real catalog data instead of empty arrays. Cross-reference `booklore-ng/BOOKLORE_ABS_IMPLEMENTATION_ISSUES.md` lines 9-100. |
| `internal/audiobooks/abs/play_response.go:119` | Replace hardcoded `currentTime: 0` with `ProgressStore.GetItemProgress(userID, profileID, libraryItemID)` lookup; emit the persisted `currentTime` and `progress` fields. |
| `internal/audiobooks/abs/libraries_handler.go:56-66` | When `?include=filterdata`, hydrate `authors`, `series`, `narrators`, `genres`, `languages`, `tags` aggregations so iOS filter UI populates. |
| `internal/audiobooks/abs/login.go` (new handler) | Add `POST /auth/refresh`. Validate refresh token type, check JTI not revoked, mint new access+refresh pair, persist new JTIs. Port from `continuum-plugin-audiobooks/internal/abs/handler.go:handleRefresh`. |
| `internal/audiobooks/abs/login.go` (new handler) | Add `POST /logout`. Marks caller's JTIs revoked in `abs_sessions`. Port from Continuum's `handleLogout`. |
| `internal/audiobooks/abs/handler.go:225` (`mountRoutes`) | Mount the new `/auth/refresh` and `/logout` routes (the latter inside `bearerAuth`). |
**Deliverable:** silo build where official ABS iOS app smoke test passes the core flow.
**Size:** ~600-800 lines across ~6 files + 2 new endpoints.
### Phase 1 — Feature surface completion
**Goal:** Bookmarks, manual collections, playlists, smart collections, RSS feeds, author/series detail, and listening stats functional in mobile clients.
**Endpoints to add** (all under `bearerAuth` unless noted; mounted at both `/abs/api/*` and `/api/*`):
**Bookmarks** (port `continuum-plugin-audiobooks/internal/abs/bookmarks_handler.go`)
- `POST /api/me/item/{itemId}/bookmark` — body `{title, time}` → array of all bookmarks for the item. Fires socket `user_updated` (`reason: "bookmark_created"`).
- `PATCH /api/me/item/{itemId}/bookmark` — upsert at time position. Fires `user_updated` (`reason: "bookmark_updated"`).
- `DELETE /api/me/item/{itemId}/bookmark/{time}` — fires `user_updated` (`reason: "bookmark_deleted"`).
**Manual Collections** (port `collections_handler.go`)
- `GET /api/collections` — list with embedded `libraryItems[]`.
- `POST /api/collections` — `{name, description}` → single collection.
- `GET/PATCH/DELETE /api/collections/{id}`.
- `POST /api/collections/{id}/book/{bookId}` — add item.
- `DELETE /api/collections/{id}/book/{bookId}` — remove item.
**Playlists** (port `playlists_handler.go`)
- `GET /api/playlists` → `{playlists: [...]}`.
- `POST /api/playlists` — `{name, description, cover_item, is_public}`.
- `GET/PATCH/DELETE /api/playlists/{id}`.
- `POST /api/playlists/{id}/item` — `{libraryItemId, episodeId?}`.
- `POST /api/playlists/{id}/batch/add` — `{items: [...]}`.
- `POST /api/playlists/{id}/batch/remove`.
- `DELETE /api/playlists/{id}/item/{libraryItemId}[/{episodeId}]`.
**Smart Collections** (port `smart_collection_handler.go` + the entire `internal/smartcoll/` DSL package; adapt SQL to silo's `media_items` schema)
- `GET /api/me/smart-collections` → `{items: [...]}`.
- `POST /api/me/smart-collections` — `{name, description, color, is_public, is_pinned, query_def}`.
- `GET /api/me/smart-collections/{id}`.
- `GET /api/me/smart-collections/{id}/items` — evaluates `query_def` against the catalog.
- `PATCH/DELETE /api/me/smart-collections/{id}`.
**Author detail**
- `GET /api/authors/{id}` — single author with `books[]`.
- `GET /api/authors/{id}/image` — wire to `item_people.poster_path` via `DetailService.PresignURL` (currently 404s).
**Series detail**
- `GET /api/series/{id}` — single series with `books[]` sorted by `series_sequence`.
**RSS Feeds** (port `rss_feed_handler.go` + `internal/podcastfeed/`)
- `POST /api/feeds` — `{itemId, slug, minified}` → `{success, feed: {id, slug, url, ...}}`.
- `GET /api/feeds` — caller's feeds.
- `DELETE /api/feeds/{id}`.
- `GET /abs/public/feed/{slug}` — public RSS XML (no auth, slug is capability token).
- `GET /abs/public/feed/{slug}/cover`.
- `GET /abs/public/feed/{slug}/item/{itemId}/{fileIdx}`.
**Listening stats** (port `internal/abs/handler.go:handleListeningStats` + `handleListeningSessions`)
- `GET /api/me/listening-stats` → `{totalTime, items, days, dayOfWeek, monthly}` from `abs_playback_sessions` aggregation.
- `GET /api/me/listening-sessions?limit=...` — paginated session history.
- `GET /api/me/listening-sessions/{sid}` — single session detail.
**Continue-listening toggles** (port `continue_listening.go`)
- `GET /api/me/progress/{itemId}/remove-from-continue-listening`.
- `GET /api/me/progress/{itemId}/readd-to-continue-listening`.
- Adds `hide_from_continue boolean` column to `user_watch_progress` (the table the ABS layer writes via `abs_progress_store.go`).
**Migrations** (numbered after silo's current max, paired up/down per `CLAUDE.md`):
- `148_abs_bookmarks` — `(id ULID PK, user_id, profile_id, library_item_id, time_seconds, title, created_at, updated_at)` + unique `(user_id, profile_id, library_item_id, time_seconds)`.
- `149_abs_user_collections` + `150_abs_collection_items`.
- `151_abs_playlists` + `152_abs_playlist_items`.
- `153_abs_smart_collections` — includes `query_def jsonb`.
- `154_abs_rss_feeds` — slug is unique capability token.
- `155_abs_progress_hide_from_continue` — adds `hide_from_continue boolean default false`.
**Deliverable:** ABS iOS/Android Library, Collections, Playlists, Bookmarks, RSS, Stats tabs all functional.
**Size:** ~3000-4000 lines across ~15 new files + 7 migration pairs. Splittable into 4 sub-commits (bookmarks, collections+playlists, smart collections, RSS+stats).
### Phase 2 — Socket.io full event parity
**Goal:** A logged-in mobile client sees real-time updates when another device on the same account changes progress, when a server scan adds items, when collections/playlists are mutated remotely, and when an admin posts notifications.
**Event surface** (union of Continuum + booklore-ng — what real ABS iOS source subscribes to):
**Server → Client** (~30 events):
- Lifecycle: `init`, `auth_failed`, `user_online`, `user_offline`, `listener_count`.
- Progress/session: `user_item_progress_updated`, `user_session_open`, `user_session_updated`, `user_session_closed`, `user_stream_update`, `user_stream_end`.
- User-scoped: `user_updated` (bookmark reasons).
- Library: `library_added`, `library_updated`, `library_removed`.
- Items: `item_added`, `item_updated`, `item_removed`, `items_added`.
- People/series: `author_added`, `author_updated`, `author_removed`, `series_added`, `series_updated`, `series_removed`.
- Collections/playlists: `collection_added/updated/removed`, `playlist_added/updated/removed`.
- RSS: `rss_feed_open`, `rss_feed_closed`.
- Tasks: `scan_start`, `scan_progress`, `scan_complete`, `task_started`, `task_progress`, `task_finished`.
- Misc: `notification`.
**Client → Server** (10 events):
- `auth` (exists), `ping`, `join_library`, `leave_library`, `playback_start`, `playback_sync`, `playback_end`, `stream_open`, `stream_close`, `sync_progress`.
**Architecture changes to `internal/audiobooks/abssocket/`:**
- Add room types: existing `user:<userID>`, new `lib:<libraryID>` (joined via `join_library`), new `admin:*`.
- New file `abssocket/publisher.go` exposing `Publisher` interface (`PublishUser`, `PublishLibrary`, `Broadcast`). Replaces ad-hoc `h.publish/h.broadcast` helpers on the Handler struct (currently silent no-ops when `SocketIO == nil`). Tests pass a recording Publisher.
**Cross-package event-source hooks** (via a new `audiobooks.EventBroadcaster` interface in `api.Dependencies` so the consumer packages don't import `abssocket` directly):
- `internal/scanner/` — emit `scan_*`, `item_*`, `items_added` on audiobook-library scan operations.
- `internal/taskmanager/` — emit `task_*` for audiobook-scoped tasks.
- `internal/api/handlers/libraries.go` (or wherever library CRUD lives) — emit `library_*`.
**Auth/lifecycle improvements:**
- Richer `init` payload: `{userId, connectedAt, libraries[], itemTagsAccessible[]}`.
- Heartbeat: respond to `ping` with `pong`.
- On disconnect, decrement listener count and broadcast `user_offline` if this was the user's last device.
- Reconnect replay (last 30s of user-scoped events) is out of scope for v1; leave a hook.
**Migrations:** none — sockets are stateless except for in-memory connection registry.
**Size:** ~1500-2000 lines: ~600 in `abssocket/`, ~400 publisher wiring across audiobook handlers, ~500 cross-package hooks.
### Phase 3 — Hardening
**Media token signing layer** (port `continuum-plugin-audiobooks/internal/mediatoken/`):
- Separate HS256 secret (`audiobooks.abs.media_signing_secret`), 15-min TTL, claims bound to `(user_id, profile_id, book_id, file_idx)`.
- `file_handler.go` mints when building stream URLs; validates on file request.
- Defense-in-depth: a leaked stream URL grants access to ONE file for 15 minutes instead of full account.
- Secret stored in `server_settings`, auto-generated on first read.
**Device tracking** (currently `abs_sessions.device_id = JTI`, useless for UI):
- Parse `User-Agent` on `/login` → derive `device_name`, `client_name`, `client_version`, store on `abs_sessions` insert.
- `GET /api/me/sessions` — list user's sessions (so they see "iPhone 15 Pro — Sep 1").
- `DELETE /api/me/sessions/{jti}` — selective revocation.
**Audit log:**
- New table `abs_audit_log`: `(id, user_id, action, ip, user_agent, metadata jsonb, created_at)`.
- Logged events: login success/failure, logout, token refresh, session revoke, smart-collection-rule changes.
- Optional `GET /api/admin/audit-log` for ops visibility.
**Rate limiting beyond login:**
- Per-user limit on `POST /me/progress` (1/sec — clients sometimes spam this).
- Per-user limit on `POST /auth/refresh` (5/hour — prevent JTI accumulation).
**Size:** ~800-1200 lines + 2 migration pairs.
## 5. Data Model
### New tables (all in Phase 1 unless noted)
```
abs_bookmarks (
id text PRIMARY KEY, -- ULID
user_id integer NOT NULL,
profile_id uuid, -- nullable: primary profile
library_item_id text NOT NULL,
time_seconds double precision NOT NULL,
title text,
created_at timestamptz DEFAULT now(),
updated_at timestamptz DEFAULT now(),
UNIQUE (user_id, profile_id, library_item_id, time_seconds)
)
abs_user_collections (
id text PRIMARY KEY, -- ULID
user_id integer NOT NULL,
profile_id uuid,
name text NOT NULL,
description text,
is_public boolean DEFAULT false,
created_at timestamptz DEFAULT now(),
updated_at timestamptz DEFAULT now()
)
abs_collection_items (
collection_id text NOT NULL REFERENCES abs_user_collections(id) ON DELETE CASCADE,
library_item_id text NOT NULL,
added_at timestamptz DEFAULT now(),
PRIMARY KEY (collection_id, library_item_id)
)
abs_playlists (
id text PRIMARY KEY,
user_id integer NOT NULL,
profile_id uuid,
name text NOT NULL,
description text,
cover_item text, -- library_item_id for cover
is_public boolean DEFAULT false,
created_at timestamptz DEFAULT now(),
updated_at timestamptz DEFAULT now()
)
abs_playlist_items (
playlist_id text NOT NULL REFERENCES abs_playlists(id) ON DELETE CASCADE,
position integer NOT NULL,
library_item_id text NOT NULL,
episode_id text, -- nullable: book item vs podcast episode
added_at timestamptz DEFAULT now(),
PRIMARY KEY (playlist_id, position)
)
abs_smart_collections (
id text PRIMARY KEY,
user_id integer NOT NULL,
profile_id uuid,
name text NOT NULL,
description text,
color text,
is_public boolean DEFAULT false,
is_pinned boolean DEFAULT false,
query_def jsonb NOT NULL,
created_at timestamptz DEFAULT now(),
updated_at timestamptz DEFAULT now()
)
abs_rss_feeds (
id text PRIMARY KEY,
user_id integer NOT NULL,
slug text UNIQUE NOT NULL, -- capability token in URL
item_id text NOT NULL,
minified boolean DEFAULT false,
created_at timestamptz DEFAULT now()
)
-- Phase 3
abs_audit_log (
id bigserial PRIMARY KEY,
user_id integer,
action text NOT NULL,
ip inet,
user_agent text,
metadata jsonb,
created_at timestamptz DEFAULT now()
)
```
### Modifications to existing tables
```
-- Phase 1: hide-from-continue toggle
ALTER TABLE user_watch_progress ADD COLUMN hide_from_continue boolean DEFAULT false;
-- Phase 3: real device tracking
ALTER TABLE abs_sessions
ALTER COLUMN device_id DROP DEFAULT,
ADD COLUMN parsed_user_agent text;
```
## 6. Testing Strategy
**Unit tests** (Go, in-package):
- Table-driven per handler in `internal/audiobooks/abs/`: happy path, missing field, wrong field type, IDOR (other user's session/bookmark/collection).
- `abssocket/` `Publisher` recording mock — verify events fire on the right operations with the right payload shape.
- Smart collection DSL evaluator — port Continuum's `smartcoll_test.go`.
**Integration tests** (against real Postgres + Redis via silo's `internal/audiobooks/testutil` harness):
- Full login → browse → play → progress → close-session flow.
- Token refresh — old token rejected, new token accepted.
- Bookmark CRUD round-trip with socket event capture.
- Smart collection rule evaluation against seeded catalog.
**End-to-end (manual)**:
- iOS official ABS app: add server → login → browse → play offline downloads → progress sync across devices.
- Plappa: same flow. Strictest client for response shape; if Plappa works, everything works.
- Android official: same.
**Migration safety:**
- Each migration includes `.down.sql`.
- New helper `scripts/test-abs-migrations.sh` runs up → down → up against a fresh DB.
## 7. Rollout
Single-shot per phase, no human review gate. **Each phase gets its own implementation plan** (separate `writing-plans` cycle) since a combined plan would be unwieldy at this size. Start with Phase 0; subsequent phases planned only after the prior phase merges and verifies.
| Phase | PR | Verify |
|---|---|---|
| Phase 0 | One PR | `make build` → restart silo → curl `/ping` → iOS app smoke test |
| Phase 1 | One PR (or 4 sub-PRs if reviewable size matters) | iOS app: Bookmarks, Collections, Playlists, Stats tabs functional |
| Phase 2 | One PR | Two devices on same account: progress sync visible in real time |
| Phase 3 | One PR | `/api/me/sessions` shows real device names; leaked stream URL expires |
**Cross-repo coordination:** silo-android and silo-apple are NOT touched. The ABS-compat surface targets OFFICIAL ABS clients; silo's own clients use silo's native API.
**Rollback:** every migration has a working `.down.sql`. New tables don't break existing functionality; silo's native audiobook API (`/api/v1/audiobooks/*`) is untouched.
## 8. Open Follow-ups (Explicitly Out of Scope)
- ABS server-side import requests (`/api/me/request`) — Continuum had this; silo has its own `internal/requests/` system. Bridging is a future decision.
- Podcast episode-download queue events (`episode_download_*`) — only relevant if silo gains podcast download capability beyond RSS refresh.
- Backup events — silo's backup story is separate.
- Watch-Together over the ABS socket — silo has `internal/watchtogether/`, deferred.
- Reconnect replay buffer for socket events — Phase 2 leaves a hook but does not implement.
## 9. References
- Source-of-truth implementation: `continuum-plugin-audiobooks` (sibling worktree). Key files: `internal/abs/handler.go`, `bookmarks_handler.go`, `collections_handler.go`, `playlists_handler.go`, `smart_collection_handler.go`, `rss_feed_handler.go`, `continue_listening.go`, `abssocket/server.go`, `mediatoken/`.
- Bug catalog for response shape: `booklore-ng/BOOKLORE_ABS_IMPLEMENTATION_ISSUES.md`.
- Canonical socket event list: `booklore-ng/src/lib/socket/events.ts`.
- ABS API documentation: `booklore-ng/AUDIOBOOKSHELF_API_DOCUMENTATION.md`.
## 10. Phase 0 — Status
**Implemented:** 2026-05-26. Plan: `docs/superpowers/plans/2026-05-26-abs-phase-0-login-and-critical-fixes.md`. All 11 plan tasks committed on `feat/audiobooks` and deployed to the production silo container. Endpoint smoke tests confirm correct mount + status codes for `/ping`, `/login`, `/auth/refresh`, `/logout`, `/me`, `/authorize` at both root and `/api`/`/abs/api` prefixes. End-to-end iOS/Plappa app validation pending operator hand-off.
**Phase 0 commits (13 total — implementation + review-feedback fixes):**
| # | SHA | Subject |
|---|-----|---------|
| 1 | `6c9c721` | feat(audiobooks): diagnostic logging to ABS bearer auth and login |
| 1+ | `d5edb24` | chore(audiobooks): normalize slog "err" key + add path to secret-fetch log |
| 2 | `877120f` | fix(audiobooks): enrich ABS login envelope |
| 2+ | `47c04f1` | chore(audiobooks): reuse existing now var in login envelope |
| 3 | `045984c` | fix(audiobooks): /authorize returns identical envelope to /login |
| 3+ | `0306cfe` | docs(audiobooks): restore x-return-tokens and displayName fallback comments |
| 4 | `e163858` | fix(audiobooks): emit IDs on authors/series and stable genres/tags arrays |
| 4+ | `470a483` | fix(audiobooks): proper pagination total + tags key in play session |
| 5 | `7f600b2` | fix(audiobooks): hydrate filterdata authors and series in library detail |
| 5+ | `2e2b411` | chore(audiobooks): rename const cap to fetchCap in buildFilterData |
| 6 | `7e956f1` | fix(audiobooks): seed currentTime from ProgressStore so resume works |
| 7 | `93e321f` | feat(audiobooks): POST /auth/refresh for ABS token rotation |
| 8 | `278f815` | feat(audiobooks): POST /logout for ABS sign-out |
**Tests added:** 15 (6 metadata + 4 resume + 5 refresh + 4 logout — note 4 logout overlaps with refresh's memTokenStore). `go test ./internal/audiobooks/... -count=1` all pass.
**Next:** Phase 1 (bookmarks, collections, playlists, smart collections, RSS, author/series detail, listening stats) will start with its own brainstorming → writing-plans cycle after operator confirms Phase 0 works against real clients.
docs/superpowers/specs/2026-05-26-abs-phase1-closeout-design.md
+344
View File
@@ -0,0 +1,344 @@
# ABS Phase 1 Close-out — Sub-Project 4
**Status:** Approved 2026-05-26. Ready for implementation plan.
**Scope:** Final sub-project of Phase 1 (per `2026-05-26-abs-implementation-fix-design.md`). Closes out the four remaining feature surfaces in one combined spec/plan to ship in a single push.
**Predecessor specs:** bookmarks, collections+playlists, smart collections. Established conventions (anti-enumeration 404, `io.LimitReader(1<<20)`, `errors.Is(err, pgx.ErrNoRows)`, profile-scoped + cross-user-public, 200-on-POST, in-package test fakes, `dispatchABSWithParams`) carry over without re-justification.
Commands assume the repository root is the cwd.
## 1. Goal
Land the four remaining Phase 1 surfaces:
- **Listening stats** — totalTime / days / dayOfWeek / monthly aggregations + paginated session history + per-session detail.
- **Author detail** — `GET /authors/{id}` and `GET /authors/{id}/image` (the latter currently 404s; wire to people poster image).
- **Series detail** — `GET /series/{id}` with embedded books sorted by series_sequence.
- **Continue-listening toggles** — `/me/progress/{itemId}/remove-from-continue-listening` + `/readd-to-continue-listening`, backed by a new `hide_from_continue` column on `user_watch_progress`.
- **RSS feeds** — minimal viable surface: open a feed for a library item, list caller's feeds, close a feed, and serve the public RSS XML to podcast clients. Series/collection feed variants deferred.
## 2. Non-goals
- **No socket events.** Continue-listening toggles, stats, author/series, and RSS feed mutations fire no events (Phase 2 covers full event parity).
- **No series/collection RSS feeds.** Only single-item feeds in v1. The continuum `/api/feeds/series/{id}/open` and `/api/feeds/collection/{id}/open` variants are explicitly deferred.
- **No RSS cover / per-track endpoints.** `GET /feed/{slug}/cover` and `GET /feed/{slug}/item/{itemId}/{idx}` are deferred — podcast clients can fetch covers from the enclosure URL embedded in the RSS XML.
- **No stable series IDs.** silo stores series denormalised on `audiobook_series.content_id` (one row per book); a "series" is identified by `LOWER(series_name)`. The detail handler accepts URL-decoded series names; URL slugs use the lowercased name. Continuum's stable series-row schema is out of scope.
- **No author write surface.** Author detail is read-only.
## 3. Architecture
Five surfaces, sharing the established structure. Three new HTTP handler files; one new migration; minimal extensions to existing stores; one new pgx store for RSS.
- **HTTP layer (`internal/audiobooks/abs/`):**
- `listening_stats_handler.go` — three handlers (stats / list / detail).
- `author_series_handler.go` — three handlers (author / author-image / series).
- `continue_listening_handler.go` — two handlers.
- `rss_feeds_handler.go` — four authenticated handlers + one public XML handler.
- **Store extensions:**
- `ABSPlaybackSessionStore.AggregateStats(ctx, userID, profileID) (Stats, error)` — single SQL with grouped aggregates.
- `ABSPlaybackSessionStore.ListClosedSessions(ctx, userID, profileID, limit, offset) ([]ABSPlaybackSession, int, error)` — paginated history.
- `ProgressStore.SetHideFromContinue(ctx, userID, profileID, contentID string, hide bool) error` — toggle column.
- `MediaStore.GetAuthorByID(ctx, authorID string) (Author, error)` — author detail + books list.
- `MediaStore.GetSeriesByName(ctx, seriesName string) (Series, error)` — series detail + books list ordered by series_index.
- **New store interface + concrete:**
- `RSSFeedStore` interface (CRUD + lookup-by-slug) in `internal/audiobooks/abs/rss_feeds.go`.
- `ABSRSSFeedStore` (pgx) in `internal/audiobooks/abs_rss_feed_store.go`.
- **Migration:**
- `154_user_watch_progress_hide_from_continue.up.sql` — `ALTER TABLE user_watch_progress ADD COLUMN hide_from_continue boolean NOT NULL DEFAULT false`.
- `155_abs_rss_feeds.up.sql` — feed rows (id, user_id, profile_id, library_item_id, slug, minified, created_at, closed_at).
- **Service wiring + route registration:** new fields on `abs.Dependencies` + new routes in `mountRoutes` (auth-gated group plus a new unauth public group for `/feed/{slug}.xml`).
## 4. Endpoint surface
All authenticated routes mount under both `/abs/api/*` and `/api/*` inside the existing `bearerAuth` group. Success status is **200 OK** except where the `Returns` column says otherwise.
### 4.1 Listening stats
| Verb | Path | Returns |
|---|---|---|
| GET | `/me/listening-stats` | `{totalTime, items, days, dayOfWeek, monthly}` aggregated from `abs_playback_sessions` |
| GET | `/me/listening-sessions` (?limit=, ?page=) | `pagedEnvelope` of session rows (most-recent-first) |
| GET | `/me/listening-sessions/{sid}` | single session detail (404 when not found or not owned) |
Stats wire shape:
```json
{
"totalTime": 36000,
"items": 12,
"days": [{"date": "2026-05-26", "seconds": 1800}, ...],
"dayOfWeek": {"0": 1200, "1": 3600, ..., "6": 900},
"monthly": [{"month": "2026-05", "seconds": 18000}, ...]
}
```
Session row shape:
```json
{
"id": "01HSESS...",
"libraryItemId": "126887...",
"userId": "1",
"startedAt": 1779786284823,
"lastUpdate": 1779786884823,
"duration": 1500,
"currentTime": 1234.5,
"timeListening": 600
}
```
### 4.2 Author + Series detail
| Verb | Path | Returns |
|---|---|---|
| GET | `/authors/{id}` | `{id, name, numBooks, books: [LibraryItem]}` |
| GET | `/authors/{id}/image` | redirect to PresignURL for `item_people.poster_path` when set; otherwise 404 |
| GET | `/series/{id}` | `{id, name, numBooks, books: [LibraryItem]}` ordered by series_index ASC |
`{id}` for authors is the stringified `people.id`. `{id}` for series is the URL-decoded series name (case-insensitive lookup). The author-image route was already mounted unauthenticated in handler.go but currently returns 404 — this spec wires the body.
### 4.3 Continue-listening toggles
| Verb | Path | Returns |
|---|---|---|
| GET | `/me/progress/{itemId}/remove-from-continue-listening` | 200 `{ok: true}` after setting `hide_from_continue = true` |
| GET | `/me/progress/{itemId}/readd-to-continue-listening` | 200 `{ok: true}` after setting `hide_from_continue = false` |
Both endpoints are idempotent. Item validation via `MediaStore.GetAudiobookByID` (404 on unknown item). The existing `MediaStore.ListContinueListening` SQL is updated to add `AND uwp.hide_from_continue = false` to the WHERE clause.
### 4.4 RSS feeds
| Verb | Path | Auth | Body | Returns |
|---|---|---|---|---|
| GET | `/api/feeds` | bearer | — | `{feeds: [Feed]}` for caller (owner + profile scope, only open feeds) |
| POST | `/api/feeds/item/{itemId}/open` | bearer | `{slug?, minified?}` | created Feed (full shape) |
| POST | `/api/feeds/{id}/close` | bearer | — | 204 (sets closed_at) |
| GET | `/feed/{slug}.xml` | none | — | RSS 2.0 XML; 404 on unknown / closed |
| GET | `/feed/{slug}` | none | — | same as above (some podcast clients omit the `.xml`) |
Feed wire shape:
```json
{
"id": "01HFEED...",
"userId": "1",
"libraryItemId": "126887...",
"slug": "abc-randomsluggenerated",
"minified": false,
"createdAt": 1779786284823,
"url": "https://silo.example.com/feed/abc-randomsluggenerated.xml"
}
```
`slug` is a 16-char URL-safe random string when caller omits it from the body. Manual slugs accepted as long as they match `^[a-z0-9-]{4,64}$`. Slugs are globally unique (UNIQUE index).
RSS XML uses standard RSS 2.0 + iTunes namespace; each `<item>` is one media_file (chapter) of the underlying audiobook. URLs are absolute (`absBaseURL` + `/public/session/...` style — but RSS feeds need a stable public path, so v1 uses a new public file route below). For v1 simplicity, the `<enclosure url>` points at the existing `/abs/public/session/{sid}/track/{idx}` style URL — but feeds don't have an associated session. So we use a new public route mounted in tandem:
**Additional public RSS support route:**
| Verb | Path | Auth | Returns |
|---|---|---|---|
| GET | `/feed/{slug}/file/{ino}` | none | streams the media file for the feed's library_item_id (404 if ino doesn't belong to the item) |
This is a feed-scoped public file route — slug is the capability token, so no Bearer needed. Mirrors the bookmarked-session pattern from sub-project 1.
## 5. Data model
### 5.1 Migration 154 — `hide_from_continue` column
`migrations/154_user_watch_progress_hide_from_continue.up.sql`:
```sql
ALTER TABLE public.user_watch_progress
ADD COLUMN IF NOT EXISTS hide_from_continue boolean NOT NULL DEFAULT false;
```
`migrations/154_user_watch_progress_hide_from_continue.down.sql`:
```sql
ALTER TABLE public.user_watch_progress
DROP COLUMN IF EXISTS hide_from_continue;
```
### 5.2 Migration 155 — `abs_rss_feeds`
`migrations/155_abs_rss_feeds.up.sql`:
```sql
CREATE TABLE IF NOT EXISTS public.abs_rss_feeds (
id text PRIMARY KEY,
user_id integer NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
profile_id uuid,
library_item_id text NOT NULL REFERENCES public.media_items(content_id) ON DELETE CASCADE,
slug text NOT NULL,
minified boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
closed_at timestamptz
);
CREATE UNIQUE INDEX IF NOT EXISTS abs_rss_feeds_slug_uniq
ON public.abs_rss_feeds (slug);
CREATE INDEX IF NOT EXISTS abs_rss_feeds_user_profile_idx
ON public.abs_rss_feeds (
user_id,
COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid)
);
```
`migrations/155_abs_rss_feeds.down.sql`:
```sql
DROP INDEX IF EXISTS public.abs_rss_feeds_user_profile_idx;
DROP INDEX IF EXISTS public.abs_rss_feeds_slug_uniq;
DROP TABLE IF EXISTS public.abs_rss_feeds;
```
### 5.3 Go models
```go
// Stats aggregates per spec §4.1.
type Stats struct {
TotalTime int // seconds
Items int // distinct content_ids listened to
Days []DayStat // last 30 days
DayOfWeek [7]int // index 0=Sunday
Monthly []MonthStat // last 12 months
}
type DayStat struct{ Date string; Seconds int } // "2026-05-26"
type MonthStat struct{ Month string; Seconds int } // "2026-05"
// Author is the detail-shape row.
type Author struct {
ID string
Name string
PosterPath string // empty when unset; resolved via CoverResolver on emit
Books []*models.MediaItem
}
// Series is the detail-shape row.
type Series struct {
ID string // lowercased series_name
Name string // canonical series_name
Books []*models.MediaItem // ordered by series_index ASC
}
// RSSFeed mirrors an abs_rss_feeds row.
type RSSFeed struct {
ID string
UserID string
ProfileID string
LibraryItemID string
Slug string
Minified bool
CreatedAt time.Time
ClosedAt *time.Time
}
```
## 6. Storage contracts
```go
// Extensions to existing interfaces:
// On ABSPlaybackSessionStore (existing):
AggregateStats(ctx context.Context, userID, profileID string) (Stats, error)
ListClosedSessions(ctx context.Context, userID, profileID string, limit, offset int) ([]ABSPlaybackSession, int, error)
// On ProgressStore (existing):
SetHideFromContinue(ctx context.Context, userID, profileID, contentID string, hide bool) error
// On MediaStore (existing):
GetAuthorByID(ctx context.Context, authorID string) (Author, error)
GetSeriesByName(ctx context.Context, seriesName string) (Series, error)
// New store interface — RSSFeedStore (defined in internal/audiobooks/abs/rss_feeds.go):
type RSSFeedStore interface {
ListUserFeeds(ctx context.Context, userID, profileID string) ([]RSSFeed, error)
GetFeedBySlug(ctx context.Context, slug string) (RSSFeed, error)
GetFeed(ctx context.Context, id string) (RSSFeed, error)
CreateFeed(ctx context.Context, f RSSFeed) error
CloseFeed(ctx context.Context, id string) error
}
```
Behavior:
- `AggregateStats` runs one SQL with three CTE subqueries (days / day_of_week / monthly aggregates) plus a `SUM(time_listening_seconds)` and `COUNT(DISTINCT content_id)`.
- `SetHideFromContinue` writes `UPDATE user_watch_progress SET hide_from_continue = $4 WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3`. Returns `nil` even when no row matched (idempotent — no-progress means nothing to hide; client can readd later).
- `GetAuthorByID` parses `authorID` as int (`strconv.Atoi`), then `SELECT name FROM people WHERE id = $1` for the author row + `SELECT mi.* FROM item_people ip JOIN media_items mi ON mi.content_id = ip.content_id WHERE ip.person_id = $1 AND ip.kind = 7 AND mi.type = 'audiobook' ORDER BY mi.title`. Returns `ErrNotFound` when the people row is missing.
- `GetSeriesByName` does case-insensitive lookup on `audiobook_series.series_name`: `SELECT DISTINCT series_name FROM audiobook_series WHERE LOWER(series_name) = LOWER($1) LIMIT 1` → name; then `SELECT mi.* FROM audiobook_series s JOIN media_items mi ON mi.content_id = s.content_id WHERE LOWER(s.series_name) = LOWER($1) AND mi.type = 'audiobook' ORDER BY s.series_index NULLS LAST`. Returns `ErrNotFound` when no rows.
- `RSSFeedStore.CreateFeed` inserts with the slug as-is; UNIQUE constraint on slug surfaces as a 409 client-side. `GetFeedBySlug` only returns rows where `closed_at IS NULL` (closed feeds 404 to podcast clients). `CloseFeed` sets `closed_at = now()` — idempotent on already-closed rows.
## 7. Error model
| Condition | Status | Body |
|---|---|---|
| 401 missing/invalid bearer | 401 | bearerAuth |
| Body decode (POST `/feeds/.../open`) | 400 | `invalid body` |
| Slug fails `^[a-z0-9-]{4,64}$` | 400 | `invalid slug` |
| Item not found on author/series/feed POST | 404 | `<type> not found` |
| Author/Series unknown ID | 404 | `<author\|series> not found` |
| RSS slug collision on POST | 409 | `slug taken` |
| Non-owner DELETE/close feed | 404 | `feed not found` (anti-enumeration) |
| Public `/feed/{slug}` unknown or closed slug | 404 | `feed not found` (plain text body — podcast clients ignore it) |
| Store mutate fails | 500 | sanitized; err logged |
Cross-cutting:
- Same `io.LimitReader(1<<20)` on POST/PATCH bodies.
- Continue-listening toggles are idempotent and return 200 even on no-row-matched.
- The public RSS routes (`/feed/{slug}.xml`, `/feed/{slug}`, `/feed/{slug}/file/{ino}`) mount OUTSIDE the bearerAuth group — they're capability-gated by the slug. Mount alongside the existing public-session routes in `mountRoutes`.
## 8. Testing
### 8.1 Listening stats
- `TestStats_TotalAggregation` — seed 3 closed sessions, assert `totalTime == sum(time_listening_seconds)` and `items == 2` (3 sessions across 2 distinct items).
- `TestStats_DayOfWeekBucketing` — sessions on a Monday + a Wednesday + another Monday, assert `dayOfWeek[1] == 2 sessions worth, dayOfWeek[3] == 1`.
- `TestSessions_List_PaginatedAndScoped` — paged response shape, other-user sessions excluded.
- `TestSession_Detail_Owner_OK` + `TestSession_Detail_NonOwner_404`.
### 8.2 Author + Series
- `TestAuthor_Detail_ReturnsBooks` — seed author with 2 books; GET returns both, name matches.
- `TestAuthor_Unknown_404`.
- `TestAuthor_Image_PresignWired` — when CoverResolver returns a URL, image route 302s; when poster_path is empty, 404.
- `TestSeries_Detail_OrderedByIndex` — books returned in series_index ASC.
- `TestSeries_Unknown_404` (name doesn't match any audiobook_series row).
### 8.3 Continue-listening
- `TestContinue_Remove_SetsHideTrue` — seed progress row, GET remove-from-continue, assert column becomes true.
- `TestContinue_Readd_SetsHideFalse` — inverse.
- `TestContinue_UnknownItem_404`.
- `TestContinue_NoProgressRow_OK_Idempotent` — no progress row exists; remove-from-continue still returns 200 (idempotent no-op).
### 8.4 RSS
- `TestFeed_Open_Item_GeneratesSlug` — POST `/api/feeds/item/{id}/open` returns a feed with a random 16-char slug.
- `TestFeed_Open_AcceptsCustomSlug`.
- `TestFeed_Open_RejectsInvalidSlug_400`.
- `TestFeed_Open_RejectsCollision_409`.
- `TestFeed_List_OwnerOnly` — only caller's open feeds, profile-scoped.
- `TestFeed_Close_Owner_204`.
- `TestFeed_Close_NonOwner_404`.
- `TestPublicFeed_UnknownSlug_404`.
- `TestPublicFeed_ClosedSlug_404`.
- `TestPublicFeed_HappyPath_RSSXML` — open a feed with a known item; fetch `/feed/{slug}.xml`; assert Content-Type is `application/rss+xml`, body contains `<rss`, `<channel>`, `<item>`, `<enclosure url=`.
### 8.5 Out of scope
- No real-Postgres SQL test.
- No live smoke for the public RSS endpoints (operator can hit them; v1 doesn't include the curl snippet).
- No RSS validator test against a third-party tool.
## 9. References
- Continuum reference: `continuum-plugin-audiobooks/internal/abs/rss_feed_handler.go` (port routing names + slug generation; adapt store/auth to silo).
- silo's existing public-route precedent: `handlePublicTrack` in `internal/audiobooks/abs/file_handler.go` — same slug-as-capability pattern.
- Item-people kinds: 7 = author per `media_store.go:ListLibraryAuthors`. Narrator kind not used here.
## 10. Out-of-scope follow-ups
- RSS feed variants: series feeds, collection feeds, cover endpoint, per-track public route is included but is the minimum.
- Listening-stats `/all-time` vs `/last-N-days` filters — current spec returns a single aggregate.
- Author edit / merge surface.
- Series metadata edit.
- Stats charts beyond the three buckets (per-genre, per-author, per-narrator) — defer.
docs/superpowers/specs/2026-05-26-abs-smart-collections-design.md
+377
View File
@@ -0,0 +1,377 @@
# ABS Smart Collections — Phase 1 Sub-Project 3
**Status:** Approved 2026-05-26. Ready for implementation plan.
**Scope:** Third sub-project of Phase 1 (per `2026-05-26-abs-implementation-fix-design.md`).
**Predecessor specs:** `2026-05-26-abs-bookmarks-design.md`, `2026-05-26-abs-collections-playlists-design.md`. Conventions established there (anti-enumeration 404, `io.LimitReader(1<<20)` body cap, `errors.Is(err, pgx.ErrNoRows)` store pattern, profile-scoped + cross-user-public read, 200 OK on POST, in-package test fakes + `dispatchABSWithParams`) are reused without re-justification.
Commands in this document assume the repository root is the cwd.
## 1. Goal
Land rule-based "smart" audiobook collections on the silo ABS surface so the official Audiobookshelf clients can define dynamic groupings ("books from 2020-2024 by Brandon Sanderson", "audiobooks I've started but haven't finished", "5-star fantasy under 12 hours"). Rules are stored as a JSON DSL; the items endpoint evaluates the rules against the catalog at request time and returns paginated `LibraryItem` results.
## 2. Non-goals
- **No SQL pushdown.** Evaluation walks the candidate list in Go. silo's audiobook library is small enough (hundreds of items) that linear scan per request is fine. SQL pushdown is a Phase 4 follow-up if the library grows past ~5000 items.
- **No rule-builder UI.** Clients ship their own; silo just stores and evaluates `query_def`.
- **No podcast smart collections.** The DSL catalog is audiobook-domain only (title/author/narrator/series/genre/...). Podcast smart collections, if ever needed, get their own DSL.
- **No socket events.** Matches the manual-collections decision in sub-project 2. Phase 2 covers full event parity.
- **No saved-search aliases.** A smart collection is fully described by its `query_def`; there's no separate "saved query" entity.
- **No background materialisation.** Each `/items` call re-evaluates. If perf matters later, add a 30s in-memory result cache keyed on `(collectionID, userID)`.
## 3. Architecture
Three layers, sharing the established silo ABS structure:
- **DSL layer (new package `internal/audiobooks/smartcoll/`):**
- `query.go` — `QueryDefinition`, `QueryGroup`, `QueryRule`, `QuerySort` types + field/sort catalogs + `Normalize`/`Validate`/`MarshalJSON`. Ported from continuum's `internal/smartcoll/query.go` verbatim except for the import path.
- `evaluator.go` — `Candidate`, `EvaluateOptions`, `Evaluate(ctx, qd, candidates, opts) []Candidate`. Pure function, no I/O. Ported from continuum.
- `query_test.go` + `evaluator_test.go` — port continuum's test suites; remove any references to continuum-specific backend types.
- **HTTP layer (`internal/audiobooks/abs/`):**
- `smart_collections_handler.go` — six handlers: `handleListSmartCollections`, `handleCreateSmartCollection`, `handleGetSmartCollection`, `handleSmartCollectionItems`, `handleUpdateSmartCollection`, `handleDeleteSmartCollection`.
- `smart_collections_handler_test.go` — in-memory `memSmartCollectionStore` + handler tests reusing `dispatchABSWithParams`, `stubMediaStore`, `recordingPublisher` from prior sub-projects' test files.
- `smart_collections_envelope_test.go` — wire-shape marshal test.
- **Storage layer:**
- `internal/audiobooks/abs/smart_collections.go` — `SmartCollectionStore` interface + `SmartCollection` Go model + `smartCollectionToABS` serialiser.
- `internal/audiobooks/abs_smart_collection_store.go` — pgx-backed `ABSSmartCollectionStore` (parallel to `abs_collection_store.go`).
- **Migration:** `153_abs_smart_collections.up.sql` + `.down.sql`.
- **Bookmark store extension:**
- `BookmarkStore.CountByUser(ctx, userID, profileID) (map[string]int, error)` returning `{libraryItemID: count}` for batch hydration of the personalized `bookmark_count` rule. Single SQL: `SELECT library_item_id, COUNT(*) FROM abs_bookmarks WHERE user_id = $1 AND COALESCE(profile_id, sentinel) = COALESCE($2, sentinel) GROUP BY library_item_id`.
- **Service wiring:** `BuildABSHandler` constructs the new store; field lands on `abs.Dependencies` next to `PlaylistStore`.
## 4. Endpoint surface
All routes mounted under both `/abs/api/*` and `/api/*` inside the existing `bearerAuth` group. Success status is **200 OK** unless the `Returns` column says otherwise.
| Verb | Path | Body | Returns |
|---|---|---|---|
| GET | `/me/smart-collections` | — | `{"items": [SmartCollection list-shape]}` |
| POST | `/me/smart-collections` | `{name, description?, color?, isPublic?, isPinned?, query_def}` | SmartCollection full-shape |
| GET | `/me/smart-collections/{id}` | — | SmartCollection full-shape when owner OR `isPublic=true`; otherwise 404 |
| GET | `/me/smart-collections/{id}/items` | (query: `?limit=`, `?page=`) | Paged envelope of `LibraryItem` results from rule evaluation |
| PATCH | `/me/smart-collections/{id}` | `{name?, description?, color?, isPublic?, isPinned?, query_def?}` | SmartCollection full-shape |
| DELETE | `/me/smart-collections/{id}` | — | 204 No Content |
### 4.1 Wire shape — SmartCollection
**List-shape and full-shape are identical** (unlike manual collections / playlists where the list-shape omits child items). Smart collections have no stored items; the items are computed by the separate `/items` route. So one shape:
```json
{
"id": "01HSC...",
"userId": "1",
"name": "Recent Fantasy",
"description": "fantasy added in the last 6 months",
"color": "#3b82f6",
"isPublic": false,
"isPinned": true,
"queryDef": {
"library_ids": [9],
"match": "all",
"groups": [{
"match": "all",
"rules": [
{"field": "genre", "op": "contains", "value": "Fantasy"},
{"field": "added_at", "op": "in_last", "value": "180d"}
]
}],
"sort": {"field": "added_at", "order": "desc"},
"limit": null
},
"createdAt": 1779786284823,
"updatedAt": 1779786284823
}
```
All ten top-level keys always present (`description`, `color` default to `""`; `isPublic`, `isPinned` default to `false`; `queryDef` is the parsed JSON, never raw bytes on the wire). Timestamps are JS-epoch milliseconds.
### 4.2 Wire shape — `/items` envelope
Standard ABS paged envelope (`pagedEnvelope` helper already in `handler.go`):
```json
{
"results": [ /* hydrated LibraryItem objects */ ],
"total": 42,
"limit": 30,
"page": 0,
"sortBy": "added_at",
"sortDesc": true,
"filterBy": "",
"minified": false,
"include": ""
}
```
`results` items use the same `siloItemToLibraryItem` shape as `/api/libraries/{id}/items`. Pagination is post-eval slice (eval order is preserved; pages are stable for a given `(collection, user)` as long as the underlying catalog doesn't change).
### 4.3 List-envelope wrap key
Smart-collection LIST uses `{"items": [...]}` — matches continuum's wrap key (which differs from manual collections' `{"collections": [...]}`). Clients pattern-match on the wrap key when distinguishing surfaces; don't override.
## 5. DSL surface (full vocabulary)
### 5.1 Fields (15 total)
**Non-personalized (10):**
| Field | Type | Valid ops | Source |
|---|---|---|---|
| `title` | scalar string | `is`, `is_not`, `contains` | `media_items.title` |
| `author` | array of strings | `is`, `is_not`, `contains` | item people: role=author |
| `narrator` | array of strings | `is`, `is_not`, `contains` | item people: role=narrator |
| `series` | array of strings | `is`, `is_not`, `contains` | `audiobook_series` |
| `genre` | array of strings | `is`, `is_not`, `contains` | `media_items.genres` |
| `year` | int | `is`, `is_not`, `gt`, `gte`, `lt`, `lte`, `between` | `media_items.release_year` |
| `rating` | float | `gt`, `gte`, `lt`, `lte`, `between` | `media_items.rating_imdb` (or whichever rating silo populates) |
| `language` | scalar string | `is`, `is_not` | `media_items.original_language` |
| `publisher` | scalar string | `is`, `is_not`, `contains` | `media_items.publisher` |
| `added_at` | timestamp | `gt`, `lt`, `between`, `in_last` | `media_items.added_at` |
| `duration_seconds` | int | `gt`, `gte`, `lt`, `lte`, `between` | sum of media_files.duration for the item |
**Personalized (5)** — require `AllowPersonalized: true` (only when caller is the collection's owner):
| Field | Valid ops | Source |
|---|---|---|
| `finished` | `is` | `user_watch_progress.is_finished` |
| `in_progress` | `is` | progress exists AND `is_finished == false` AND `current_seconds > 0` |
| `last_played` | `gt`, `gte`, `lt`, `lte`, `between`, `in_last` | `user_watch_progress.updated_at` |
| `abandoned` | `is` | `in_progress == true` AND `last_played < now() - 60d` (configurable via `EvaluateOptions.AbandonedAfter`) |
| `bookmark_count` | `gt`, `gte`, `lt`, `lte`, `between` | count of `abs_bookmarks` rows for (user, profile, item) |
### 5.2 Sort fields
`title` (asc), `added_at` (desc), `year` (desc), `duration_seconds` (desc), `rating` (desc), `random` (deterministic shuffle seeded by `userID + ":" + collectionID`), and personalized: `progress` (desc), `last_played` (desc), `plays` (desc).
### 5.3 Operators
- `is` / `is_not` — equality (case-insensitive for strings; numeric otherwise). For array fields, `is` matches when ANY element equals the value.
- `contains` — substring (case-insensitive) for scalar strings; element-substring for array fields.
- `gt` / `gte` / `lt` / `lte` — numeric or timestamp comparison.
- `between` — `value` is a 2-element array `[low, high]`, inclusive bounds.
- `in_last` — relative window. `value` is a duration string (`"7d"`, `"180d"`, `"24h"`, `"4w"`). Evaluator parses to `time.Duration`; field must be ≥ `opts.Now - duration`.
### 5.4 Aliases
`authors` → `author`, `narrators` → `narrator`, `genres` → `genre` for fields. `sort_title` → `title`, `recently_added` → `added_at`, `duration` → `duration_seconds` for sorts. Applied in `Normalize()`; older clients with plural / verbose names keep working.
## 6. Data model
### 6.1 Migration 153
`migrations/153_abs_smart_collections.up.sql`:
```sql
CREATE TABLE IF NOT EXISTS public.abs_smart_collections (
id text PRIMARY KEY,
user_id integer NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
profile_id uuid,
name text NOT NULL,
description text NOT NULL DEFAULT '',
color text NOT NULL DEFAULT '',
is_public boolean NOT NULL DEFAULT false,
is_pinned boolean NOT NULL DEFAULT false,
query_def jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS abs_smart_collections_user_profile_idx
ON public.abs_smart_collections (
user_id,
COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid)
);
```
`migrations/153_abs_smart_collections.down.sql`:
```sql
DROP INDEX IF EXISTS public.abs_smart_collections_user_profile_idx;
DROP TABLE IF EXISTS public.abs_smart_collections;
```
### 6.2 Schema rationale
- **`query_def jsonb`** — JSONB so future GIN-indexed JSON path probes are an option; default `'{}'::jsonb` so rows with no rules are queryable without NULL guards.
- **No items table** — smart collections have no stored children; items are computed at eval time.
- **`color`, `is_pinned`** — UI decorations. silo doesn't validate the color format.
- **No `library_ids` column** — that list is part of the DSL, lives inside `query_def`.
### 6.3 Go model
```go
type SmartCollection struct {
ID string
UserID string
ProfileID string
Name string
Description string
Color string
IsPublic bool
IsPinned bool
QueryDef []byte // raw JSONB bytes; decoded on the items path
CreatedAt time.Time
UpdatedAt time.Time
}
```
## 7. Storage contract
```go
type SmartCollectionStore interface {
ListUserSmartCollections(ctx context.Context, userID, profileID string) ([]SmartCollection, error)
GetSmartCollection(ctx context.Context, id string) (SmartCollection, error)
CreateSmartCollection(ctx context.Context, c SmartCollection) error
UpdateSmartCollection(ctx context.Context, c SmartCollection) error
DeleteSmartCollection(ctx context.Context, id string) error
}
```
Behavior follows the established silo pattern: ordered by `created_at DESC`, empty slice never nil, `GetSmartCollection` returns `ErrNotFound` without owner check (handler authorizes), `Update` sets `updated_at = now()`, `Delete` returns nil even on no-match.
**Extension to existing `BookmarkStore`:**
```go
// CountByUser returns a map of library_item_id → bookmark count for
// the given (user, profile). Empty map (never nil) when none.
// Used by the smart-collection items evaluator to hydrate the
// personalized `bookmark_count` rule in one SQL pass.
CountByUser(ctx context.Context, userID, profileID string) (map[string]int, error)
```
SQL: `SELECT library_item_id, COUNT(*) FROM abs_bookmarks WHERE user_id = $1 AND COALESCE(profile_id, sentinel) = COALESCE($2::uuid, sentinel) GROUP BY library_item_id`.
## 8. Items evaluation flow
`handleSmartCollectionItems` runs:
1. Auth + store-nil + URL `{id}` checks (same boilerplate as other handlers).
2. `GetSmartCollection(ctx, id)` → 404 on ErrNotFound OR `(non-owner AND !isPublic)`.
3. Decode `c.QueryDef` → `smartcoll.QueryDefinition`. 500 on malformed (shouldn't happen — `Validate` is called on persist; defensive guard).
4. Resolve target libraries: `qd.LibraryIDs` if non-empty, else every audiobook library from `MediaStore.ListAudiobookLibraries`.
5. For each target library, fetch the audiobook list via `MediaStore.ListAudiobooks(ctx, libID, 5000, 0)`. (5000 cap matches continuum's; silo's libraries are smaller.)
6. **Build Candidates.** For each item:
- `Candidate.Item` = the `*models.MediaItem`.
- If `c.UserID == a.UserID` (owner): hydrate per-user state.
- Fetch all progress rows: `ProgressStore.ListProgressForAudiobooks(ctx, a.UserID, a.ProfileID, 10000)`. Build `map[contentID]ProgressRow`.
- Fetch bookmark counts: `BookmarkStore.CountByUser(ctx, a.UserID, a.ProfileID)`. Map `contentID → count`.
- Populate `IsFinished`, `ProgressPct`, `CurrentSeconds`, `LastPlayedAt`, `BookmarkCount`.
- Non-owner viewing public collection: skip hydration; personalized rules eval against zero-values.
7. `smartcoll.Evaluate(ctx, qd, candidates, EvaluateOptions{AllowPersonalized: c.UserID == a.UserID, UserSeed: a.UserID + ":" + c.ID, Now: time.Now(), AbandonedAfter: 60*24*time.Hour})` → matched + sorted.
8. Paginate: `limit, page := readPagedQuery(r, 30)`. Override default limit with `qd.Limit` when `?limit=` query param is absent. Slice `matched[page*limit : (page+1)*limit]`.
9. Hydrate each result via the existing `siloItemToLibraryItem` helper (same as `/api/libraries/{id}/items`).
10. Wrap in `pagedEnvelope`. Write 200.
### 8.1 Performance budget
- Library fetch: one SQL query per library (~hundreds of rows).
- Progress hydration: one SQL query for the user's full progress list.
- Bookmark hydration: one SQL query for the aggregated counts.
- Total: 2 + N (libraries) queries, all paginated/limited at the store layer. silo's current 1-library config means 3 total queries per `/items` request. Fast enough.
## 9. Error model
| Condition | Status | Body |
|---|---|---|
| Missing/invalid bearer | 401 | handled by `bearerAuth` middleware |
| Body decode failure (POST/PATCH) | 400 | `invalid body` |
| `name` missing on POST | 400 | `name required` |
| `query_def` fails `Validate(allowPersonalized=true)` | 400 | `invalid query_def: <reason>` (reason from validator — safe to surface) |
| Unknown collection on GET (owner or not) | 404 | `smart collection not found` |
| Non-owner GET on private | 404 | same body — no leak |
| Non-owner PATCH/DELETE/items | 404 | same body |
| Store mutate fails | 500 | `smart collection persist failed` (delete: `smart collection delete failed`); err logged via `slog.Error` |
| `/items` eval — `MediaStore.ListAudiobooks` fails for a library | log `slog.Warn` and skip that library | partial results beat 500 |
| Personalized rule on non-owner-public eval | silently dropped at `smartcoll.Evaluate` via `AllowPersonalized: false` | no error to the client |
**Cross-cutting:** Body size limit `io.LimitReader(r.Body, 1<<20)` on all POST/PATCH. `validate(allowPersonalized=true)` runs on every persist call; on read, the items handler passes `AllowPersonalized = (c.UserID == a.UserID)` so saved personalized rules are non-fatally dropped when viewed by non-owners.
## 10. Testing
### 10.1 DSL unit tests (`internal/audiobooks/smartcoll/`)
Port continuum's `query_test.go` and `evaluator_test.go` verbatim. The continuum reference covers:
- `Normalize`: aliasing, lowercase, dedupe library_ids, default-match `"all"`, default-sort `"added_at"`.
- `Validate`: unknown field, invalid op for field, personalized rule without scope, invalid sort field, invalid sort order, negative limit.
- `Evaluate`: each operator for each field type, AND/OR combinator for groups and definition, personalized rules drop when `AllowPersonalized: false`, `random` sort stability per `UserSeed`, `Limit` honored, empty rules match-everything.
### 10.2 Handler tests (`smart_collections_handler_test.go`)
In-memory `memSmartCollectionStore` (parallel to `memCollectionStore`); reuses `dispatchABSWithParams`, `stubMediaStore`, `recordingPublisher`. The bookmark store extension (`CountByUser`) needs a matching method on `memBookmarkStore` from sub-project 1's test file — extend that fake.
| Test | Asserts |
|---|---|
| `SmartCollection_Create_ReturnsFullShape` | POST → 200, 10 top-level keys present, ULID, queryDef round-tripped as a nested object (not raw bytes) |
| `SmartCollection_Create_NameRequired_400` | empty name → 400 |
| `SmartCollection_Create_InvalidBody_400` | malformed JSON → 400 |
| `SmartCollection_Create_InvalidQueryDef_400` | `{"groups":[{"rules":[{"field":"nonsense","op":"is","value":1}]}]}` → 400 with reason in body |
| `SmartCollection_Create_PersonalizedRuleAllowed` | rule with `field:"finished"` accepted (persist always runs with `allowPersonalized=true`) |
| `SmartCollection_List_WrappedAsItems` | GET → `{"items": [...]}` envelope key (NOT `"collections"`) |
| `SmartCollection_List_DoesNotLeakOtherUsers` | user 1 creates; user 2 lists → empty |
| `SmartCollection_List_ProfileIsolation` | profile A creates; profile B lists same user → empty |
| `SmartCollection_Get_Owner_ReturnsFullShape` | owner GET → 200 + full-shape |
| `SmartCollection_Get_NonOwner_Public_OK` | public GET by other user → 200 |
| `SmartCollection_Get_NonOwner_Private_404` | private GET by other user → 404 (anti-enumeration) |
| `SmartCollection_Get_Unknown_404` | unknown ID → 404 |
| `SmartCollection_Patch_Owner_UpdatesFields` | partial PATCH updates only present fields |
| `SmartCollection_Patch_NonOwner_404` | non-owner PATCH → 404, original untouched |
| `SmartCollection_Patch_InvalidQueryDef_400` | PATCH with bad query_def → 400 |
| `SmartCollection_Delete_Owner_204` | DELETE → 204; subsequent GET → 404 |
| `SmartCollection_Delete_NonOwner_404` | non-owner DELETE → 404, original still present |
| `SmartCollection_Items_Owner_EvaluatesRules` | seed 3 audiobooks (two match the rule, one doesn't); GET /items returns 2 results |
| `SmartCollection_Items_PersonalizedDroppedForNonOwner` | public collection with `field:"finished"` rule — non-owner sees ALL matching books (personalized rule silently dropped); owner sees only finished |
| `SmartCollection_Items_PaginatedEnvelope` | GET /items → standard pagedEnvelope with 9 fields |
| `SmartCollection_Items_RespectsQueryDefLimit` | qd.Limit=5; GET without ?limit returns 5; GET with ?limit=10 returns 10 (request overrides) |
| `SmartCollection_Items_NonOwner_Private_404` | private collection /items by non-owner → 404 |
| `SmartCollection_Envelope_HasRequiredKeys` | marshal-test for 10 top-level keys with empty defaults |
### 10.3 Live integration smoke (operator)
```bash
TOKEN=$(curl ... /login | jq -r .accessToken)
# Create a smart collection: "fantasy added in the last 6 months"
SC=$(curl -s -X POST -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
-d '{"name":"recent fantasy","query_def":{"match":"all","groups":[{"match":"all","rules":[{"field":"genre","op":"contains","value":"Fantasy"},{"field":"added_at","op":"in_last","value":"180d"}]}],"sort":{"field":"added_at","order":"desc"}}}' \
http://127.0.0.1:13378/api/me/smart-collections | jq -r .id)
# Evaluate
curl ... /api/me/smart-collections/$SC/items | jq '.results | length'
# Patch name
curl -X PATCH ... -d '{"name":"renamed"}' /api/me/smart-collections/$SC | jq .name
# Delete
curl -X DELETE ... /api/me/smart-collections/$SC # 204
```
### 10.4 Out of scope
- No real-Postgres SQL test (same posture as prior sub-projects).
- No GIN-index on `query_def`. Phase 4 follow-up if rule-querying becomes a feature.
- No performance test. Library is small; linear scan is fine.
## 11. Risks & open questions
- **5000-item per-library cap.** If a library exceeds this, the over-fetch silently truncates. silo's libraries are well below this today; if it changes, switch to paged ListAudiobooks + accumulate.
- **Personalized rule eval against non-owners is silent.** A non-owner viewing a public collection sees results that don't honor the owner's personalized rules. This is the privacy-correct behavior, but a confused user might wonder why "books I haven't finished" shows everything when viewed by their friend. Document in the parent spec's UX notes.
- **`random` sort + pagination.** Two requests with the same `(user, collection)` produce the same shuffle (seeded). Browser-back / page-2 stays consistent. If the catalog mutates between requests, the seed still works but new items land in unpredictable positions — acceptable.
- **`query_def` migration on field-catalog changes.** If silo renames a field or removes an op in the future, existing rows persist the old vocab. `Validate` would reject them, but Normalize won't drop them — we'd surface stale rules as a runtime warning in the items handler. Acceptable for v1.
- **`description` length.** No cap; relies on the 1 MiB body limit. Same posture as other surfaces.
## 12. Out-of-scope follow-ups
- SQL-pushdown evaluator for large libraries.
- GIN-indexed JSON path queries on `query_def`.
- Background materialisation cache for hot smart collections.
- Cross-collection composition ("collection A + collection B").
- Result count badges on `/me/smart-collections` (would require running eval on every list call — expensive).
- Audiobook recommender hooks (the `relevance` sort placeholder in continuum).
## 13. References
- Spec parent: `docs/superpowers/specs/2026-05-26-abs-implementation-fix-design.md`.
- Sibling sub-project specs: `2026-05-26-abs-bookmarks-design.md`, `2026-05-26-abs-collections-playlists-design.md`.
- DSL reference: `continuum-plugin-audiobooks/internal/smartcoll/{query,evaluator}.go` + matching tests (port verbatim).
- HTTP reference: `continuum-plugin-audiobooks/internal/abs/smart_collection_handler.go`.
docs/superpowers/specs/2026-05-27-unified-audiobook-collections-design.md
+295
View File
@@ -0,0 +1,295 @@
# Unified User Collections (Audiobooks + Movies + TV) Design
**Status:** design — pending implementation plan.
**Source brainstorm:** in-session, 2026-05-27. Driven by audiobook coverage gaps in `page_sections`, `library_collections`, and `user_personal_collections`, plus a parallel `abs_*` collection/playlist/smart-collection stack that doesn't intersect with the silo-canonical tables.
**Commands assume the repository root is the cwd.**
---
## 1. Problem
Audiobooks today live in a parallel collections world from movies and TV:
| Surface | Audiobook coverage today |
|---|---|
| `page_sections` (homepage rails) | None. Recipes reference only `movie`/`series`. |
| `library_collections` (admin-owned, library-scoped) | Schema supports `MediaAudiobook` kind but nothing surfaces audiobook-typed admin collections in practice. |
| `user_personal_collections` (user-owned) | Zero references to audiobooks anywhere in `internal/usercollections/`. |
| `abs_user_collections` + `abs_collection_items` | Full audiobook-only user-curated collections behind the ABS-compat API. |
| `abs_smart_collections` | Audiobook-only smart collections with a rule DSL (port of continuum-plugin-audiobooks). |
| `abs_playlists` + `abs_playlist_items` | Audiobook + podcast-episode ordered playlists. |
Net effect: audiobooks don't appear in any of silo's first-party discovery surfaces, the ABS-app's collection/playlist features don't intersect with silo's native ones, and we maintain two parallel storage stacks.
The data inside the `abs_*` tables today is essentially empty (1 playlist, 0 user collections, 0 smart collections), so unification can land via hard cutover with minimal data migration cost.
---
## 2. Goals
- One canonical storage for user-owned lists across all media types.
- Audiobook items eligible for `page_sections` recipes, `library_collections`, and `user_personal_collections`.
- ABS-compat endpoints (`/api/collections`, `/api/playlists`, `/api/smart-collections`) continue to work and return identical wire shapes, but their persistence collapses into the canonical store.
- One smart-collection engine evaluated against all media types.
- Podcast-episode-level granularity preserved (ABS playlists can name a specific episode within a library item).
## 3. Non-goals
- Admin-curated audiobook collections via the ABS API surface. `library_collections` already supports audiobooks; whether ABS clients should see those admin lists is a separate question deferred for a later design.
- Frontend changes. Audiobook detail/library pages already exist (per `2026-05-24-audiobook-ui-redesign`) and will consume the unified endpoints when they ship. An admin section-builder media-type selector and any UI surfacing of audiobook sections/collections are follow-up plans.
- Cross-user shared collections, public sharing, social features. Out of scope.
- Episode-level granularity for movies/TV. The new `sub_item_id` column is nullable and only populated by podcast playlists.
## 4. Architecture
### 4.1 Storage model
One canonical user-owned-lists table — `user_personal_collections` — extended to discriminate four flavors via `collection_type`:
| `collection_type` | Semantics |
|---|---|
| `manual` | Existing. Hand-curated list of items. Order optional. |
| `synced` | Existing. External sync (Trakt etc.). Items derived from external source. |
| `playlist` | **New.** Ordered sequence; `position` column on items is meaningful. |
| `smart` | **New.** Rule-based; items materialized at read time from `query_definition` via `internal/smartcoll/`. |
`user_personal_collection_items` gains one nullable column:
```sql
ALTER TABLE user_personal_collection_items ADD COLUMN sub_item_id text NOT NULL DEFAULT '';
```
`sub_item_id` is populated only when the entry refers to a sub-item of a library item (today: podcast episode within a podcast library item). All other entries leave it empty. Audiobook entries always use empty `sub_item_id` because an audiobook is a single `media_items` row.
The CHECK constraint on `collection_type` is widened to admit the new values:
```sql
ALTER TABLE user_personal_collections
DROP CONSTRAINT IF EXISTS user_personal_collections_type_check;
ALTER TABLE user_personal_collections
ADD CONSTRAINT user_personal_collections_type_check
CHECK (collection_type IN ('manual', 'synced', 'playlist', 'smart'));
```
Admin-curated lists (`library_collections`, `library_collection_items`, etc.) are unchanged. They already support audiobooks through the existing `MediaKind` enum.
### 4.2 Smart-collection engine
The current audiobook-only smart-collection engine in `internal/audiobooks/smartcoll/` is promoted to `internal/smartcoll/`:
```
internal/audiobooks/smartcoll/ → internal/smartcoll/
```
Rationale: smart collections are no longer audiobook-only. The DSL operates on `media_items` regardless of type; audiobook-specific rule kinds (e.g. `narrator`, `series_position`) stay registered but evaluate to no-op predicates on non-audiobook items.
The existing simple-shape `query_definition` on `user_personal_collections` (mostly `library_ids` filter) is a strict subset of the smartcoll DSL — existing rows continue to evaluate correctly because their library-id filter maps to the new DSL's library-id predicate.
### 4.3 ABS-compat handler mapping
The three ABS store adapter files in `internal/audiobooks/` get their bodies rewritten to query the canonical tables. Method signatures stay the same so the ABS HTTP handlers above them don't change.
| ABS endpoint | Backing query after cutover |
|---|---|
| `GET /api/libraries/{id}/collections` | `SELECT … FROM user_personal_collections WHERE collection_type='manual'` joined to items in this library |
| `POST /api/collections` | INSERT `user_personal_collections` with `collection_type='manual'`, scoping via `query_definition.library_ids` |
| `GET /api/libraries/{id}/playlists` | `SELECT … FROM user_personal_collections WHERE collection_type='playlist'` joined to items in this library |
| `POST /api/playlists` | INSERT `user_personal_collections` with `collection_type='playlist'` |
| `GET /api/libraries/{id}/smart-collections` | `SELECT … FROM user_personal_collections WHERE collection_type='smart'`; items materialized via `internal/smartcoll` |
| `episode_id` in playlist items wire shape | maps to `sub_item_id` column |
### 4.4 Page sections (homepage rails)
`page_sections` rows gain a media-types filter. The cleanest carrier is a typed column:
```sql
ALTER TABLE page_sections
ADD COLUMN media_types text[] NOT NULL DEFAULT ARRAY['movie','series'];
```
Default preserves the current "movies and TV" behavior on existing rows so the migration doesn't change any user-visible rails.
Each recipe declaration in `internal/sections/recipes/` gains a `SupportedMediaTypes` field. Fetchers add `WHERE mi.type = ANY($media_types)` to their queries. Existing recipes set `SupportedMediaTypes = ['movie','series']`; they keep their current behavior. Recipes that work for audiobooks declare `['movie','series','audiobook']` (or `['audiobook']` for audiobook-only ones).
Two new recipes ship at the same time:
- **`continue_listening`** — analog of `continue_watching`; audiobook items with non-zero progress and not finished.
- **`by_audiobook_series`** — book series rail, drawing from the `audiobook_series` table; mirrors `by_show`.
More audiobook-flavored recipes (top narrators, by genre, new from followed authors) are deferred until the baseline two are exercised.
### 4.5 ABS app contract preservation
Wire-shape compatibility is part of the contract — the ABS Android/iOS apps cannot break. The store-adapter rewrites must produce byte-for-byte identical JSON for the existing endpoints. Concretely:
- IDs remain stable for the moved row(s). The migration preserves `abs_playlists.id` as the new `user_personal_collections.id`.
- `episode_id` field is emitted from `sub_item_id` (empty string when null/empty).
- `is_public` field on playlists/smart collections is emitted from `is_shared`.
- `cover_item` on playlists — needs decision (see §6).
### 4.6 Module structure after the change
```
internal/
smartcoll/ -- NEW (promoted from audiobooks/smartcoll/)
types.go
evaluator.go
rule_registry.go
usercollections/ -- existing; extended for playlist/smart kinds
listing.go -- gains Kind filter
types.go -- adds CollectionKind = playlist|smart
smartfetch.go -- new; materializes smart-collection items via internal/smartcoll
sections/
recipes/
*.go -- each gains SupportedMediaTypes
continue_listening.go -- new
by_audiobook_series.go -- new
audiobooks/
abs_collection_store.go -- rewritten: canonical SQL
abs_playlist_store.go -- rewritten: canonical SQL
abs_smart_collection_store.go -- rewritten: canonical SQL + smartcoll
smartcoll/ -- REMOVED (lifted to internal/smartcoll/)
```
---
## 5. Migration
Single pair: `migrations/156_unify_user_collections.up.sql` / `.down.sql`. Hard cutover.
### 5.1 Up
```sql
-- 1. Extend canonical items table with sub-item granularity.
ALTER TABLE user_personal_collection_items
ADD COLUMN sub_item_id text NOT NULL DEFAULT '';
-- 2. Widen the collection_type enum.
ALTER TABLE user_personal_collections
DROP CONSTRAINT IF EXISTS user_personal_collections_type_check;
ALTER TABLE user_personal_collections
ADD CONSTRAINT user_personal_collections_type_check
CHECK (collection_type IN ('manual', 'synced', 'playlist', 'smart'));
-- 3. Move the single existing abs_playlists row.
INSERT INTO user_personal_collections
(id, user_id, profile_id, name, description, collection_type,
is_shared, created_at, updated_at, creator_profile_id)
SELECT
id, user_id, COALESCE(profile_id::text, ''), name, description, 'playlist',
is_public, created_at, updated_at, COALESCE(profile_id::text, '')
FROM abs_playlists;
INSERT INTO user_personal_collection_items
(user_id, collection_id, media_item_id, sub_item_id, position, added_at)
SELECT p.user_id, i.playlist_id, i.library_item_id, i.episode_id, i.position, i.added_at
FROM abs_playlist_items i
JOIN abs_playlists p ON p.id = i.playlist_id;
-- 4. Drop the abs_* collection tables.
DROP TABLE abs_playlist_items;
DROP TABLE abs_playlists;
DROP TABLE abs_collection_items;
DROP TABLE abs_user_collections;
DROP TABLE abs_smart_collections;
-- 5. Extend page_sections with media-type filter.
ALTER TABLE page_sections
ADD COLUMN media_types text[] NOT NULL DEFAULT ARRAY['movie','series'];
```
### 5.2 Down
```sql
-- Reverse the page_sections column.
ALTER TABLE page_sections DROP COLUMN media_types;
-- Recreate the abs_* tables empty by re-applying their CREATE TABLE
-- statements verbatim from the original migration files (no data restored):
-- migrations/149_abs_user_collections.up.sql → abs_user_collections
-- migrations/150_abs_collection_items.up.sql → abs_collection_items
-- migrations/151_abs_playlists.up.sql → abs_playlists
-- migrations/152_abs_playlist_items.up.sql → abs_playlist_items
-- migrations/153_abs_smart_collections.up.sql → abs_smart_collections
-- The implementer copies the CREATE TABLE bodies (and indexes) into this
-- down migration; constraint/index names must match the originals so a
-- subsequent up-down-up cycle is idempotent.
-- Remove rows we promoted.
DELETE FROM user_personal_collection_items
WHERE collection_id IN (
SELECT id FROM user_personal_collections WHERE collection_type IN ('playlist','smart')
);
DELETE FROM user_personal_collections WHERE collection_type IN ('playlist','smart');
-- Restore narrow CHECK constraint.
ALTER TABLE user_personal_collections
DROP CONSTRAINT IF EXISTS user_personal_collections_type_check;
ALTER TABLE user_personal_collections
ADD CONSTRAINT user_personal_collections_type_check
CHECK (collection_type IN ('manual', 'synced'));
-- Drop the sub_item_id column.
ALTER TABLE user_personal_collection_items DROP COLUMN sub_item_id;
```
The down migration is symmetrically structured but lossy in reverse: rolling back loses any playlist/smart rows created after the cutover. That's acceptable because the abs_* tables were near-empty before the up.
### 5.3 Risk
| Risk | Probability | Mitigation |
|---|---|---|
| Down-migration loses real user data | Low (one row today; grows over time) | Snapshot the table before running down in any prod context |
| ABS app sees wire-shape regressions | Medium | Snapshot tests on JSON output of each ABS endpoint before/after |
| smartcoll DSL gap (silo-native query_definition row that doesn't fit DSL subset) | Low | Pre-migration query enumerates non-conforming rows; spec asserts subset compat |
| `library_collections` UI gains audiobook rows it can't render | Low | UI changes deferred; admin doesn't currently create audiobook-typed library collections |
---
## 6. Open questions
1. **Playlist `cover_item` field.** `abs_playlists.cover_item` is a FK to `media_items.content_id`. `user_personal_collections.poster_url` is a plain text URL. Two options for the migration: (a) drop `cover_item`, regenerate poster URLs from the first item in the playlist; or (b) add a `cover_content_id` column to `user_personal_collections`. Recommend (a) for simplicity — the existing playlist row has no `cover_item` set.
2. **`is_pinned` on smart collections.** `abs_smart_collections.is_pinned` has no analog in `user_personal_collections`. Likely needs a new boolean column; defer until smart-collection UI work names a use case.
3. **`color` on smart collections.** `abs_smart_collections.color` has no analog. Same call as `is_pinned` — defer; the existing 0 rows means no data to preserve.
4. **Admin-curated audiobook collections in the ABS API.** Out of scope per §3, but flagged for a future design.
---
## 7. Testing
- **Migration round-trip test.** Run `up.sql` → `down.sql` → `up.sql` on a fresh test DB seeded with one of each `abs_*` row. Assert canonical-table contents are identical before and after.
- **ABS endpoint wire-shape snapshots.** Generate JSON for `GET /api/libraries/{id}/collections|playlists|smart-collections` before the cutover; assert byte-equal after.
- **smartcoll engine portability.** Move the existing `internal/audiobooks/smartcoll/*_test.go` into `internal/smartcoll/` and add three new tests:
- audiobook-specific rule (`narrator` filter) evaluated against an audiobook → expected items.
- audiobook-specific rule evaluated against a movie → empty set (no-op predicate).
- simple library-id filter (silo-native query_definition shape) evaluated under the unified DSL → expected items.
- **Section recipe regression.** Existing recipes with default `media_types=['movie','series']` return identical results to today. New `continue_listening` and `by_audiobook_series` recipes covered with focused unit tests.
---
## 8. Sub-projects
This design is decomposed for implementation:
1. **Schema migration + canonical storage extensions** — migration 156, CHECK/column additions, no behavior changes yet.
2. **Smart-collection engine lift** — move `internal/audiobooks/smartcoll/` to `internal/smartcoll/`, update imports, no logic changes.
3. **ABS store adapter rewrites** — collection/playlist/smart-collection stores query canonical tables.
4. **Section recipes for audiobooks** — recipe `SupportedMediaTypes` field, two new recipes, fetcher type filter.
5. **Documentation + follow-up tickets** — open-question call-outs (cover_item, is_pinned, color, admin collections in ABS).
Each sub-project has a single-MR scope and ships independently of the others. Sub-project 1 lands first; sub-projects 2–4 can land in any order after it.
---
## 9. Source references
- Existing canonical tables: `internal/usercollections/`, `internal/collections/templates/templates.go`
- Existing smart-collection engine: `internal/audiobooks/smartcoll/`
- Existing section recipes: `internal/sections/recipes/`, `internal/api/handlers/sections_preview.go`, `internal/api/handlers/sections_bulk.go`, `internal/api/handlers/recipes.go`
- Existing ABS store adapters: `internal/audiobooks/abs_collection_store.go`, `internal/audiobooks/abs_playlist_store.go`, `internal/audiobooks/abs_smart_collection_store.go`
- Related shipped specs: `docs/superpowers/specs/2026-05-26-abs-smart-collections-design.md`, `docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md`
docs/superpowers/specs/2026-06-01-request-multi-instance-arr-routing-design.md
+353
View File
@@ -0,0 +1,353 @@
# Multi-Instance Sonarr/Radarr Routing for Requests (Seerr-style)
**Status:** Design approved, ready for implementation plan
**Date:** 2026-06-01
**Area:** `internal/requests`, `migrations/`, `web/src` (admin settings + request queue)
## Summary
Silo's request system today fulfills every approved request through a single
Radarr (movies) or single Sonarr (series) instance, at whatever quality that
one instance's profile is configured for. There is no 4K concept, no anime
handling, and exactly one instance per kind (`request_integrations.kind` is the
primary key).
This work replicates Seerr's Sonarr/Radarr management model *inside* Silo's own
request system (no external Seerr dependency):
- **Many instances per kind** — N Radarr and N Sonarr servers, each with its own
connection, root folder, quality profile, and tags.
- **Default HD + Default 4K routing** — one HD-default and one 4K-default
instance per kind; requests route to the matching default(s).
- **Per-server anime overrides** (Overseerr-style) — each instance carries an
optional anime profile / root folder / tags applied when a title is detected
as anime.
- **Entitlement-driven dual-quality fan-out** — a single user request can be
fulfilled in **both** 1080p and 2160p when the requester's `MaxPlaybackQuality`
allows 4K, or when an admin global toggle forces it for everyone.
The end-user request action is unchanged: quality is derived from the user's
existing entitlement, not a per-request choice.
## Goals
- Support multiple Radarr/Sonarr instances per kind.
- Route by quality (HD vs 4K) to per-kind default instances.
- Fan a single request out to both 1080p and 2160p targets based on the
requester's `MaxPlaybackQuality` entitlement, with an admin override that
forces dual-quality regardless of role.
- Apply per-instance anime overrides using Seerr-exact anime detection.
- Keep the request lifecycle (quotas, approval, reconcile) coherent under
one-to-many fulfillment.
## Non-Goals
- No integration *with* a Seerr instance (an earlier idea, dropped). Silo
remains the system of record.
- No per-request manual server/profile picker (Overseerr's advanced UI). Routing
is automatic from defaults + entitlement.
- No separate 1080p/4K *availability* tracking. The pre-request presence check
stays binary ("is this title in the Silo library at all").
- No admin-configurable anime keyword list (the heuristic is isolated so one can
be added later).
- No dedicated-anime-instance flag — anime is handled by per-instance overrides
(decided during design).
## Background: current flow
When a user requests a movie today (`internal/requests/service.go`):
1. **Browse/discover** via TMDB; each result carries a `RequestState`.
2. `CreateRequest` validates: requests enabled, not already available
(presence), not already actively requested (dedup by `tmdb_id`), computes the
requester's `EffectivePolicy` (quota, blocked, auto-approve), enforces quota.
3. Auto-approve users with a configured integration → row created `approved`,
else `pending` for the admin queue.
4. `submitApprovedRequest` → `integrationForMediaType(mediaType)` finds the
single instance by kind → `radarr.Client.SubmitMovie` POSTs to Radarr using
that instance's root folder / quality profile / tags → request goes `queued`.
5. `ReconcileRequests` polls the instance: `queued → downloading → completed`
(or `failed`).
The single insertion point that changes is step 4's instance selection.
## Design
### 1. Data model
**`request_integrations` — reworked from one-row-per-kind to many instances:**
| column | purpose |
|---|---|
| `id` (text PK, via `idgen`) | replaces `kind` as PK |
| `kind` (text, `radarr`\|`sonarr`) | now a plain column, keeps its CHECK |
| `name` (text) | admin-facing label |
| `enabled` (bool) | |
| `base_url`, `api_key_ref` | connection; API key stays a Fernet secret ref, never plaintext |
| `root_folder`, `quality_profile_id`, `tags[]` | standard (non-anime) defaults |
| `is_4k` (bool) | marks this as a 4K server |
| `is_default` (bool) | the HD default for its kind |
| `is_default_4k` (bool) | the 4K default for its kind |
| `anime_enabled` (bool) | apply anime overrides on this instance |
| `anime_quality_profile_id`, `anime_root_folder`, `anime_tags[]` | anime overrides |
| `options` (jsonb) | kind-specific extras (Radarr min-availability, Sonarr season-folders/series-type) |
| `last_check_*`, `created_at`, `updated_at` | unchanged |
**Invariants** (partial unique indexes + service validation):
- At most one `is_default` per kind; at most one `is_default_4k` per kind.
- `is_default ⇒ NOT is_4k` and `is_default_4k ⇒ is_4k`. The HD default is an HD
server; the 4K default is a 4K server. HD and 4K defaults are therefore always
distinct instances.
**New `media_request_targets`:**
| column | purpose |
|---|---|
| `id` (bigint identity PK) | |
| `request_id` (text → `media_requests` ON DELETE CASCADE) | |
| `integration_id` (text → `request_integrations` ON DELETE SET NULL) | |
| `integration_kind` (text snapshot) | survives instance deletion for history |
| `quality` (text, `1080p`\|`2160p`) | reuses `internal/access` presets |
| `is_anime` (bool) | which profile set was applied |
| `external_id`, `external_status`, `status`, `last_error` | per-target Radarr/Sonarr lifecycle |
| `created_at`, `updated_at` | |
| UNIQUE `(request_id, quality)` | one target per quality per request |
**`media_requests` changes:** the per-fulfillment columns
(`integration_kind`, `external_id`, `external_status`) move out to
`media_request_targets`. Add `is_anime` (bool). The request's `status` becomes an
**aggregate** over its targets (see §3). `idx_media_requests_active_tmdb` stays —
dedup is still one active request per title regardless of quality, and a
dual-quality request counts as **one** row against the user's quota.
### 2. Routing engine
A pure function `routeTargets(req, entitlement, settings, instances) -> []plannedTarget`
replaces `integrationForMediaType` as the selection point in
`submitApprovedRequest`.
**Inputs:** the request (incl. detected `is_anime`), the requester's
`MaxPlaybackQuality`, the global `force_dual_quality` setting, the configured
instances for the kind.
**Algorithm:**
1. **Kind** = `movie → radarr`, `series → sonarr`.
2. **Desired qualities:**
- `1080p` is always desired (baseline for everyone).
- Add `2160p` if `access.QualityAllowed("2160p", user.MaxPlaybackQuality)`
**OR** `force_dual_quality` is on (the toggle overrides role).
3. **Pick the default instance per desired quality:** `1080p → is_default`,
`2160p → is_default_4k` (for that kind).
4. **Profile selection per chosen instance:** if `req.is_anime` **and** the
instance has `anime_enabled` → use its anime profile / root folder / tags
(and, for Sonarr, `seriesType: anime`); otherwise the standard set.
5. Emit one `plannedTarget{instance, quality, isAnime, profile, rootFolder, tags}`
per resolved quality.
**Edge cases:**
- A desired quality with **no default instance assigned** is silently skipped
(cannot route 4K without a 4K default — matches "when both are assigned").
- **Zero** resolved targets (e.g. no HD default configured): the request stays
`approved` but unfulfilled, with `last_error` = "no Radarr/Sonarr instance
configured for this quality" so it surfaces in the admin queue.
- No dedup needed: the `is_default`/`is_default_4k` invariants guarantee HD and
4K defaults are distinct instances.
### 3. Fulfillment & reconcile lifecycle
The adapter interfaces (`SubmitMovie`/`SubmitSeries`,
`CheckMovieStatus`/`CheckSeriesStatus`) are **unchanged** — they already take
`(req, integration)`. We call them **once per target**, passing a *resolved*
`Integration` whose `RootFolder`/`QualityProfileID`/`Tags` are filled from the
standard or anime block. Radarr/Sonarr's notion of "4K" is just that instance's
quality profile + root folder, so adapters stay quality-agnostic. (The Sonarr
adapter additionally sets `seriesType: anime` for anime targets.)
**On approval (`submitApprovedRequest`):**
1. `routeTargets(...)` → planned targets.
2. For each: insert a `media_request_targets` row, then call the matching
adapter with the resolved instance + decrypted key.
3. On success → target `queued` with its `external_id`/`external_status`; on
error → target `failed` + `last_error`.
4. **Recompute the request aggregate** in the same transaction as the target
writes.
**Reconcile loop (`ReconcileRequests`):** for each candidate request, iterate its
**non-terminal targets**, load each target's instance by `integration_id`, call
the status adapter, update the target, then recompute the aggregate. Targets
reconcile independently.
**Aggregate recompute (single source of truth):**
- all targets `completed` → request `completed` (+ `completed_at`)
- any target `downloading` → `downloading`; else any `queued` → `queued`
- all targets `failed` → `outcome = failed`; **some** failed while others active
→ request stays active, `last_error` set from the failed target(s)
**Retry** is target-aware: an admin retry re-submits only the `failed` targets
(re-routing if instance config changed), leaving healthy targets alone.
### 4. Anime detection (Seerr-exact)
Detection matches upstream Seerr exactly — a single TMDB keyword id, no genre or
language fallback:
- `server/api/themoviedb/constants.ts`: `ANIME_KEYWORD_ID = 210024`
- `server/entity/MediaRequest.ts`: anime ⇔ `keywords.results` contains that id.
Implementation:
- A named constant `animeKeywordID = 210024`.
- `detectAnime(detail) bool` → true iff the TMDB keyword **ids** contain
`210024`. Matching by id (not the name string) avoids false positives from
neighboring keywords like "based on anime".
- Detected once at `CreateRequest` time and stored on `media_requests.is_anime`,
so routing, reconcile, and retries stay consistent.
- Applies to both movies and series; ignored if no instance for that kind has
`anime_enabled`.
**Supporting TMDB-client change:** extend `tmdb.MediaDetail` to carry keyword
**ids** (add `append_to_response=keywords` to the detail fetch). `GetMediaDetail`
is already called in the request path, so no extra round-trip beyond the append.
(`OriginalLanguage` is not needed under the parity-only decision but may be
mapped opportunistically since it is already in the response.)
The heuristic is isolated in one function so an admin-configurable keyword list
can be added later without touching routing.
### 5. Admin UI
`web/src/pages/admin-settings/IntegrationsSettings.tsx` and the setup wizard
`web/src/pages/setup-wizard/steps/IntegrationsStep.tsx` move from a single
Radarr/Sonarr form to an **instance-list manager** per kind:
**Per kind — list of instance cards** (add / edit / delete):
- Connection: name, base URL, API key (write-only; shows "configured" once set,
never echoes the secret), **Test connection** → `LoadIntegrationOptions`
populates root-folder and quality-profile **dropdowns**.
- Standard block: root folder, quality profile, tags.
- Quality role: `is_4k` switch, **Default (HD)** toggle, **Default 4K** toggle.
The form enforces invariants client-side (no Default-HD on a 4K server;
selecting a new default clears the prior one visually); server is source of
truth.
- Anime block (collapsible, `anime_enabled`): anime quality profile, anime root
folder, anime tags — dropdowns from the same test-connection options.
**Request settings panel** gains the **"Always fulfill in both 1080p and 4K"**
toggle (`force_dual_quality`) with helper text: applies to all requests when both
a Default HD and Default 4K instance exist, regardless of user role.
**`web/src/pages/AdminRequests.tsx`** (queue): each request row expands to show
its **targets** — quality badge (1080p/2160p), instance name, per-target status,
and per-target **retry** on failed ones. The aggregate status stays the headline.
**Setup wizard** stays minimal: add one Radarr + one Sonarr, auto-marked Default
HD. Advanced multi-instance/4K/anime config lives in the full settings page.
### 6. API contract & multi-repo coordination
**End-user request flow is unchanged** — quality is derived from
`MaxPlaybackQuality`, so the request action and payload are identical (no 4K
toggle, no new permission).
**`Request` response model** gains `is_anime` and a `targets[]` array
(`quality`, `instance_name`, `status`, `external_status`, `last_error`); the
top-level `integration_kind`/`external_id`/`external_status` are removed in favor
of the aggregate `status` + `targets`.
- Per `CLAUDE.md`'s multi-repo rule: end-user `silo-android` / `silo-apple`
clients show a single aggregate status, so they likely need **no change** —
but this is flagged as explicit Apple/Android follow-up to verify the request
model tolerates the removed fields and ignores/parses the new `targets`.
**Admin endpoints** change from "upsert by kind" to instance CRUD:
- `GET/POST/PUT/DELETE` request integrations **by `id`**; `LoadIntegrationOptions`
(test-connection) stays, keyed by instance.
- Default-toggle handled server-side transactionally (a new HD/4K default clears
the prior one for that kind).
- `request_settings` gains `force_dual_quality`.
- Admin-web-only; no mobile client surface.
### 7. Migration & backfill
Next migration number: **169** (`169_request_multi_instance.{up,down}.sql`).
**Up:**
1. `ALTER request_integrations`: add `id`, `name`, `is_4k`, `is_default`,
`is_default_4k`, `anime_enabled`, `anime_quality_profile_id`,
`anime_root_folder`, `anime_tags[]`. Backfill existing rows: `id = gen`,
`name = initcap(kind)`, `is_default = enabled` (the lone existing instance
becomes that kind's HD default), everything else default/false. Swap the PK
from `kind` to `id`; keep `kind` as a plain column with its CHECK.
2. Partial unique indexes: one `is_default` per kind, one `is_default_4k` per
kind.
3. `CREATE media_request_targets` (per §1).
4. Backfill targets: for any `media_requests` row with an `external_id`, insert a
target `(quality='1080p', is_anime=false, integration_id = that kind's
instance, integration_kind, external_id, external_status, status = request's
current status)`. Rows never submitted get no target yet.
5. `ALTER media_requests`: add `is_anime bool DEFAULT false`; drop
`integration_kind`, `external_id`, `external_status`.
6. `idx_media_requests_active_tmdb` stays unchanged.
**Down:** re-add the three columns to `media_requests` and copy back the **1080p**
target's external fields; drop `media_request_targets`; drop `is_anime`; collapse
`request_integrations` to `kind`-PK by keeping each kind's `is_default` instance
and discarding extras; drop the new columns/indexes.
**Caveat (explicit):** the down migration is **lossy** — rollback discards any
additional instances (beyond one default per kind) and any 4K/anime target
history. This is the unavoidable cost of collapsing a one-to-many back to
one-to-one and is acceptable for a down migration.
**Go-side data access** (implementation, not SQL): `Integration` struct +
`scanIntegration` gain the new fields; the `Store` interface gains target CRUD +
`ListInstances(kind)`; `scanRequest` drops the external fields; `UpsertIntegration`
becomes id-based CRUD with default-toggle handling in-transaction.
### 8. Testing
- **`routeTargets` — table-driven (core of the feature):** matrix of
{1080p vs 2160p entitlement} × {`force_dual_quality` on/off} × {which defaults
exist} × {anime vs not} → expected target set.
- **`detectAnime`:** keyword `210024` present/absent; no name-substring false
positives.
- **Aggregate recompute:** all-complete, partial-failure (request stays active +
`last_error`), all-failed (`outcome=failed`), mixed downloading/queued.
- **Repository:** target CRUD; single-default-per-kind invariant rejects a second
default; migration backfill yields exactly one 1080p target per previously
submitted request.
- **Adapter:** Sonarr `SubmitSeries` sets `seriesType=anime` only for anime
targets on anime-enabled instances.
- **Service:** extend `service_test.go` fakes — a 4K-entitled user's single
request fans out to two adapter submissions with the right resolved instances;
target-scoped retry re-submits only the failed target.
- **Frontend:** instance-form default-invariant enforcement; queue renders
per-target rows + retry.
## Risks & open considerations
- **Reconcile cost** scales with targets, not requests (≤2× today). Bounded and
acceptable; the reconcile loop already batches.
- **Anime keyword imperfection** is inherited from Seerr by design (parity
choice). Mitigated by isolating `detectAnime` for future tuning.
- **Client model drift**: removing top-level external fields from `Request` must
be verified against `silo-android` / `silo-apple` deserialization before
release (flagged in §6).
## References
- Seerr upstream: https://github.com/seerr-team/seerr
(`server/api/themoviedb/constants.ts`, `server/entity/MediaRequest.ts`)
- Overseerr anime discussions: #2876 (Sonarr for anime only), #3777 (configurable
series type)
- Existing code: `internal/requests/{service,repository,types,store}.go`,
`internal/requests/{radarr,sonarr,arrclient}`, `internal/access/quality.go`,
`migrations/139_media_requests.up.sql`
docs/superpowers/specs/2026-06-02-autoscan-arr-polling-design.md
+222
View File
@@ -0,0 +1,222 @@
# Autoscan: Polling Radarr/Sonarr to Trigger Targeted Library Scans
**Status:** Design approved, ready for implementation plan
**Date:** 2026-06-02
**Area:** new `internal/autoscan`, `internal/taskmanager/tasks`, `migrations/`, `internal/api`, `web/src/pages/AdminRequests.tsx`
## Summary
Silo currently learns about new media via scheduled/manual library scans. When
Radarr/Sonarr import a download, the new file only appears in Silo on the next
scan. This adds an **autoscan poller**: a periodic background task that polls the
configured Radarr/Sonarr instances for newly-imported files and enqueues
**targeted folder scans** into Silo's existing scan queue, so requested (and
non-requested) content shows up promptly.
The design follows the established autoscan pattern — poll arr import history,
translate the imported paths, and trigger targeted scans — but kept deliberately
simple. That pattern is normally built to orchestrate scans across many separate
media-server boxes, which needs a bounded fan-out guard and a retry queue. Silo
is a single service scanning *itself*, so the cross-node fan-out machinery is
unnecessary — imported paths become folder jobs in Silo's own worker-bounded
`scanqueue`.
## Goals
- Poll autoscan-enabled Radarr/Sonarr instances on an interval for import events.
- Translate arr import paths into Silo media folders and enqueue targeted scans.
- Reuse the Radarr/Sonarr instances already configured for requests
(`request_integrations`) — no duplicate URLs/keys.
- Coalesce bursts (a season's episodes) into minimal redundant scans.
- Admin-configurable: global enable, interval, debounce, per-instance toggle,
optional path rewrites, on-demand "poll now".
## Non-Goals
- No cross-node scan fan-out or bounded-concurrency semaphore (single service;
the `scanqueue` workers are the bound).
- No separate retry queue (a failed source poll simply does not advance its
high-water mark, so the next cycle retries the window).
- No arr webhook/"Connect" push integration (polling only, as requested; webhook
push is a possible future add-on).
- No separate autoscan source credential store — sources are existing
`request_integrations`.
## Background
- **`internal/scantrigger`** already maps a filesystem path to a Silo
`MediaFolder` and scan path: `Resolver.Resolve(Request{Path}) -> Target{Folder,
Mode, Path}` (via `LongestMatchingRoot` / `MatchFolderForPath`).
- **`internal/scanqueue`** executes scans through a worker-bounded queue:
`Queuer.EnqueueScan(ctx, folderID, mode, path, trigger)` /
`EnqueueScans(targets)`.
- **`internal/requests/arrclient`** is a thin Radarr/Sonarr HTTP client
(`New(baseURL, apiKey, httpClient)`, `GetJSON`, `DoJSON`) — reused for the new
history call.
- **`request_integrations`** holds each instance's `id`, `kind`, `base_url`, and
Fernet-encrypted `api_key_ref`, resolved via the same secret resolver the
request service uses.
- The **task manager** runs periodic tasks via the `taskmanager.Task` interface
with interval `DefaultTriggers()` (e.g. `ReconcileRequestsTask`).
## Design
### 1. Data model & config
A new `internal/autoscan` package owns its own schema; `request_integrations`
stays focused on fulfillment.
**`autoscan_settings`** (singleton, mirrors `request_settings`):
| column | type / default | purpose |
|---|---|---|
| `id` | bool PK default true, singleton CHECK | |
| `enabled` | bool, default false | master on/off |
| `poll_interval_minutes` | int, default 10 | poll task interval |
| `debounce_seconds` | int, default 60 | folder re-scan suppression window |
| `created_at`, `updated_at` | timestamptz | |
**`autoscan_sources`** (one row per autoscan-enabled instance):
| column | type | purpose |
|---|---|---|
| `integration_id` | text PK → `request_integrations(id)` ON DELETE CASCADE | the source instance |
| `enabled` | bool, default false | per-instance autoscan toggle |
| `path_rewrites` | jsonb, default `[]` | `[{from, to}]` prefix overrides |
| `last_poll_at` | timestamptz, null | `history/since` high-water mark |
| `created_at`, `updated_at` | timestamptz | |
**Reused, not duplicated:** `base_url` / `api_key_ref` come from
`request_integrations`. Debounce/suppression state lives in **Redis**
(`autoscan:scanned:{folder_id}`, TTL = `debounce_seconds`), not Postgres.
### 2. Poll cycle
`autoscan.Service.PollOnce(ctx)` (invoked by the periodic task):
1. Load `autoscan_settings`; if `!enabled`, return immediately (no-op).
2. Load `autoscan_sources WHERE enabled`, joined to `request_integrations` for
`kind` / `base_url` / `api_key_ref`.
3. For each source (sequential — a handful of instances):
1. Decrypt the API key via the secret resolver.
2. Record `cycleStart := now`.
3. `arrclient.GetJSON("/api/v3/history/since?date={last_poll_at}&eventType=downloadFolderImported")`
— both Radarr and Sonarr expose this. `last_poll_at` null → a sane floor
(do not scan all history on first enable; floor at `cycleStart`).
4. Extract the imported **file path** per event (Radarr: movie file path;
Sonarr: episode file `outputPath`); take its **parent directory**.
5. On success, advance that source's `last_poll_at = cycleStart`. On failure
(arr unreachable, decode error), log and **do not** advance — next cycle
retries the window.
4. For each extracted directory: apply path rewrites (§3), resolve via
`scantrigger.Resolver` → `Target{Folder, Mode, Path}`. Unresolvable paths
(outside Silo's media folders) are logged and skipped.
5. **Dedupe to unique folders** for this cycle; for each, check the Redis
suppression key — if absent, `EnqueueScan(..., trigger="autoscan")` and set
the key with TTL `debounce_seconds`; if present, skip (already scanned this
window).
**Debounce rationale:** because the poller batches a whole interval of imports
per `history/since` call, within-cycle folder dedup does most of the work. The
Redis suppression key only prevents re-enqueuing the same folder on back-to-back
cycles. Silo's scanner is idempotent, so a season importing across two cycles
simply gets a second (cheap) scan that picks up the rest. This is simpler than a
"collect-expired" debounce plus a retry queue, which the polling + idempotent
re-scan model makes unnecessary.
### 3. Path resolution & scan enqueue
A small pipeline in front of the existing scan plumbing — no new scan logic:
1. **Rewrite** (per source, optional): apply the first matching `path_rewrites`
entry as a prefix replacement (e.g. `/data/media -> /mnt/media`). No rewrites
→ path used as-is (shared-mount common case).
2. **Resolve** via `scantrigger.Resolver.Resolve(Request{Path: rewrittenDir})`
→ `Target{Folder, Mode, Path}`. No matching media folder → log + skip.
3. **Enqueue** deduped targets via `scanqueue` `EnqueueScans` with
`trigger="autoscan"` (attributable in scan history, flows through the same
worker-bounded queue).
### 4. Scheduling & concurrency
- **`internal/taskmanager/tasks/autoscan_poll.go`** implements `taskmanager.Task`
(`Key: "autoscan_poll"`, name/description/category, `DefaultTriggers` = interval
trigger seeded from `poll_interval_minutes`). Its run body calls
`autoscan.Service.PollOnce(ctx)`.
- **Disabled = cheap no-op:** the task stays registered; `PollOnce` returns early
when `enabled` is false. Toggling autoscan never (de)registers tasks.
- **Interval changes:** updating `poll_interval_minutes` reconfigures the task's
interval trigger via the task manager's existing trigger-config path; if a live
update is unavailable it applies on next start.
- **No overlap, no semaphore:** the task manager serializes a task's executions;
sources are polled sequentially; scan execution is bounded by `scanqueue`. The
single-service simplification: no bounded-fan-out semaphore is needed.
- **Primary/integrated server only** (same as the request reconcile task), not on
standalone transcode/proxy worker nodes.
### 5. Admin API & UI
**API** (admin-gated, `/api/v1/admin/autoscan`, mirroring the request-settings
handlers; never echoes API keys):
- `GET/PUT /autoscan/settings` — `{enabled, poll_interval_minutes, debounce_seconds}`
- `GET /autoscan/sources` — per autoscan-eligible instance (joined from
`request_integrations`): `{integration_id, name, kind, enabled, path_rewrites,
last_poll_at}`
- `PUT /autoscan/sources/{integration_id}` — set `{enabled, path_rewrites}`
- `POST /autoscan/trigger` — run `PollOnce` once on demand; tight per-admin rate
limit so it cannot hammer the arr instances
- `GET /autoscan/status` — `{enabled, per-source last_poll_at, last scan summary}`
**UI:** a new **"Autoscan" tab** in `web/src/pages/AdminRequests.tsx` (alongside
Queue / Settings / Integrations / User Overrides):
- Global controls: enable switch, poll interval, debounce window.
- Per-source list — one row per Radarr/Sonarr instance with an autoscan toggle,
an optional `from → to` path-rewrite editor, and a read-only "last polled".
- A **"Poll now"** button → `POST /autoscan/trigger`.
Rate limits and admin-auth follow Silo's existing request-admin conventions.
### 6. Testing
- **Path rewrite:** prefix replacement, first-match ordering, no-match
passthrough, trailing-slash handling.
- **arr history client:** table-driven `httptest` fixtures for both Radarr and
Sonarr import-event JSON shapes; asserts the correct file path is extracted and
its parent folder taken.
- **Dedupe + suppression:** many paths in one folder → one target; folder with a
live suppression key skipped; absent/expired key enqueues.
- **`PollOnce` (integration seam):** fakes for source store, arr history client,
`scantrigger.Resolver`, `scanqueue.Queuer`, Redis — asserts deduped folder
scans enqueued with `trigger="autoscan"`, `last_poll_at` advanced on success,
**not** advanced on per-source failure, clean no-op when disabled.
- **Unresolvable path:** logged + skipped, never enqueued or fatal.
- **Repository / migration:** `autoscan_settings` singleton + `autoscan_sources`
CRUD and the `request_integrations` join; migration applies + rolls back
(sidecar table, `ON DELETE CASCADE`).
- **Frontend:** Autoscan tab renders, per-source toggle / path-rewrite editor
round-trips, "Poll now" calls the trigger endpoint.
## Risks & open considerations
- **First-enable floor:** `last_poll_at` null must floor at "now" (not epoch) so
enabling autoscan doesn't enqueue a scan for every historical import.
- **history/since shape differences:** Radarr vs Sonarr import-event payloads
differ; the path extraction is per-kind and covered by fixtures.
- **Path rewrites vs scantrigger roots:** if neither rewrites nor Silo's media
folders match the arr path, the import is silently skipped (logged) — this is
the expected "content outside Silo's libraries" behavior, but a misconfigured
rewrite will look like "autoscan does nothing"; the `last_poll_at` + status
endpoint and skip logs are the debugging surface.
- **Idempotent re-scan:** relies on Silo's scanner being safe to run repeatedly
on a folder (it is — re-scan finds new files).
## References
- Existing Silo plumbing reused by this design: `internal/scantrigger/scantrigger.go`,
`internal/scanqueue/`, `internal/requests/arrclient/`,
`internal/taskmanager/tasks/reconcile_requests.go`
- Prior art: the general autoscan pattern (poll arr import history → translate
paths → trigger targeted scans). Silo's variant drops the cross-node fan-out
guard and retry queue, which only a multi-box orchestrator needs.
docs/superpowers/specs/2026-06-02-autoscan-plugin-architecture-design.md
+262
View File
@@ -0,0 +1,262 @@
# Autoscan as a Pluggable Scan-Source Category — Design Spec
**Date:** 2026-06-02
**Status:** Approved design, pre-implementation
**Goal:** Turn autoscan from a Requests-coupled, arr-only feature into a standalone **Autoscan** category whose change-detection providers are out-of-process Silo plugins, with Sonarr/Radarr as the first provider.
**Architecture (one sentence):** Silo's host keeps a thin, provider-agnostic engine that resolves changed paths to library folders and enqueues rescans; each *way of noticing change* (arr, later inotify/ceph) is an installable plugin that Silo pulls on a timer through a new additive `scan_source.v1` capability.
**Tech stack:** Go (silo-server host + plugins), `silo-plugin-sdk` (protobuf capability contract, consumed as a tagged module), existing `internal/plugins` + `internal/pluginhost` runtime, existing `scantrigger`/`scanqueue` scan pipeline, React/TS admin UI.
Commands assume the repository root is the cwd.
---
## 1. Background & motivation
The current autoscan implementation (delivered in the now-closed PR #43, still on local `main`) is in-process Go that:
- polls Sonarr/Radarr `/api/v3/history/since` on a host timer,
- rewrites arr-side paths to Silo paths, dedups to parent dirs,
- resolves each to a `MediaFolder` and enqueues a targeted scan,
- is configured against `request_integrations` rows (hard FK, `ON DELETE CASCADE`), living as a tab inside the **Requests** admin page.
Two structural problems:
1. **It is arr-only and not extensible.** Adding inotify, Ceph notifications, or any other change source means more bespoke host code.
2. **It is subordinate to Requests.** A source cannot exist without a Requests integration row; deleting the row destroys the source.
Silo already ships a full out-of-process plugin runtime (`internal/plugins`, `internal/pluginhost`): manifests, an installer, typed capability descriptors (`metadata_provider.v1`, `request_router.v1`, `media_analyzer.v1`, `scheduled_task.v1`, `event_consumer.v1`, …), a host-driven scheduled-task mechanism, and a rate-limited plugin→host event path. This design uses that runtime to make change-detection pluggable.
## 2. Goals / non-goals
**Goals**
- A new **Autoscan** admin category, independent of Requests.
- A new additive `scan_source.v1` plugin capability — **breaks no existing plugin**.
- A **client-pull** contract: Silo asks each source "what changed since `<marker>`?" on a timer.
- Sonarr/Radarr re-implemented as the first scan-source plugin; the closed PR's arr logic relocates into it.
- A provider-agnostic host engine reusable by every future scan source.
- Optional reuse of Requests' arr connection details, *or* standalone connection entry.
**Non-goals (this design)**
- inotify / Ceph / other providers (the contract must *accommodate* them; we do not build them).
- A push (plugin-initiated) delivery mode. Pull only; push sources buffer internally and answer on pull (see §6).
- Refactoring the Requests subsystem itself.
## 3. Architecture overview
```
┌──────────────────────────── Silo (host) ────────────────────────────┐
│ Autoscan category (new; not under Requests) │
│ • Sources: which scan_source plugin instance, on/off, interval │
│ • Connections: reuse a Requests arr server, or enter own │
│ • Per-source state: opaque marker, last-run status │
│ │ │
│ ▼ │
│ AutoscanStore — generic engine (provider-agnostic) │
│ per tick, per enabled source: │
│ call plugin PollChanges(marker) │
│ → resolve each Silo-native path → MediaFolder │
│ → suppress duplicates → enqueue scan (existing scanner) │
│ → store returned next_marker │
│ │ scan_source.v1 (pull) │
└──────────────────────────────┼───────────────────────────────────────┘
▼
┌──────── Sonarr/Radarr scan-source plugin ────────┐
│ (new installable repo, like silo-plugin-tmdb) │
│ receives resolved arr address + key │
│ polls /history (imports + renames) │
│ returns RAW source paths + next marker │
│ (host applies the source's rewrites) │
└──────────────────────────────────────────────────┘
```
**Five components**
1. **`scan_source.v1`** (in `silo-plugin-sdk`) — the pull contract. §5.
2. **AutoscanStore** (host) — generic resolve→suppress→enqueue→marker engine. §7.
3. **Connection resolution** (host) — reference-or-inline → concrete credentials. §8.
4. **Autoscan category + data model** (host) — sources, connections, decoupled from Requests. §9.
5. **arr scan-source plugin** (new repo) — first provider. §10.
## 4. Responsibility split (the dividing line)
| Concern | Owner | Rationale |
|---|---|---|
| Talk to arr; poll `/history`; imports + renames; bounded window; first-run | **Plugin** | Provider-specific I/O. |
| **Path rewrites** (per-source config + application) | **Host** | _(Revised post-merge: the SDK landed host-owned rewrites.)_ The plugin returns RAW source-namespace paths in `source_paths`; the host applies each source's rewrite rules (longest-`From`-first) before resolving. Keeps every provider dumb and rewrite behavior uniform. Rewrites are edited per-source in the Autoscan UI and stored in `autoscan_sources.path_rewrites`. |
| Resolve Silo-native path → `MediaFolder`; dedupe; **enqueue rescan** | **Host** | Touches Silo internals (scan queue, DB, library map) that plugins cannot and must not reach. |
| Opaque per-source marker; poll interval; enabled; last-run status | **Host** | Scheduling and bookkeeping are the engine's job. |
| Connection selection (reuse-from-Requests vs own) and credential resolution | **Host** | Only the host knows about Requests; the plugin receives resolved creds. |
The plugin's responsibility ends at **"here are the changed paths, in Silo's terms."** Everything past that is the host engine, which is provider-agnostic.
## 5. The `scan_source.v1` contract (centerpiece)
A new, additive capability in `silo-plugin-sdk`. Existing plugins never declare it and are unaffected (§11).
### Protobuf (as implemented in silo-plugin-sdk)
```proto
// scan_source.v1 — host pulls changed paths from a provider on a timer.
service ScanSource {
// Host calls this on each poll tick for a configured scan_source capability.
rpc PollChanges(PollChangesRequest) returns (PollChangesResponse);
}
message PollChangesRequest {
string capability_id = 1; // which configured scan_source instance
string marker = 2; // opaque; empty string on first run
ResolvedConnection connection = 3; // resolved upstream credentials (see §8)
}
// Concrete upstream credentials the host resolves and hands to the plugin each
// poll, so the plugin stays credential-source-agnostic and stores no secrets.
message ResolvedConnection {
string base_url = 1;
string api_key = 2;
}
message PollChangesResponse {
// Absolute changed paths in the provider/source namespace (RAW — NOT yet
// rewritten). Files or directories. The host applies the source's rewrite
// rules and validates before enqueueing.
repeated string source_paths = 1;
// Opaque continuation token. Host stores verbatim and echoes it back
// on the next PollChanges. The plugin never assumes the host parses it.
string next_marker = 2;
}
```
### Contract rules
- **Pull only.** The host owns the timer and calls `PollChanges`. The plugin never initiates.
- **Opaque marker.** The host stores and echoes `next_marker` without interpreting it. arr uses an RFC3339 timestamp; a future provider may use a sequence number or event id — no host change required.
- **First run.** `marker == ""` ⇒ the plugin starts from "now" (do not replay full history).
- **Idempotent, at-least-once.** The host may re-issue the same `marker` if it failed to persist `next_marker`; providers must tolerate returning overlapping paths. The host's suppression layer (§7) absorbs duplicates.
- **Error semantics.** A failed `PollChanges` (e.g. arr unreachable) ⇒ the host keeps the *old* marker and retries next tick; nothing is skipped or advanced.
- **Raw source paths.** `source_paths` are in the provider's namespace, NOT rewritten. The host applies each source's path-rewrite rules (host-owned, per-source) before resolving to a library folder.
- **Credentials per call.** The host resolves the source's connection (§8) and passes the concrete `{base_url, api_key}` in `PollChangesRequest.connection` on every poll. The plugin reads them from the request — it stores no credentials and never knows whether they came from its own config or a reused Requests link. Delivered over the local host↔plugin gRPC channel only.
The capability needs **only** `PollChanges` for v1. A `TestConnection`/validate RPC is explicitly deferred (YAGNI; connection validity surfaces as a failed poll).
## 6. Marker & delivery semantics
- One opaque marker stored per source, host-side.
- Pull-only does **not** exclude push-style providers: an inotify/Ceph plugin maintains its own internal buffer of observed changes and flushes it (and advances its marker) on each `PollChanges`. Cost: up to one poll-interval of latency. This keeps a single uniform contract.
## 7. Host: AutoscanStore engine
Provider-agnostic. Per enabled source, per tick:
1. `paths, next := plugin.PollChanges(marker)`; on error, log + keep marker + continue.
2. For each `dir` in `uniqueParentDirs(paths)`:
- `target := scantrigger.Resolve(dir)`. Unresolvable (outside Silo's media folders) ⇒ quiet skip.
- Suppression key `fmt.Sprintf("%d|%s", folderID, target.Path)` with debounce TTL; already-claimed ⇒ skip.
- Accumulate `target`; record claimed key.
3. `scanqueue.EnqueueScans(targets)`. On failure, release claims and **do not** advance the marker.
4. On success, persist `next_marker`.
This is the salvaged core of PR #43 (`internal/autoscan/service.go` resolve/suppress/enqueue, `dedupe.go`, `suppress.go`), generalized so its input is "paths from a provider" rather than "paths from arr." Path-rewrite helpers (`rewrite.go`, `suggest.go`, `history.go`, the arr history client) leave the host and move into the plugin (§10).
## 8. Host: Connections (reuse or own)
A **connection** is "an arr server Silo can reach," created two ways:
- **Own:** name + base URL + API key, entered in the Autoscan category; the key is Fernet-encrypted at rest (same mechanism as existing service credentials — never plaintext in JSONB).
- **Linked (reuse):** a live reference to an existing Requests `request_integrations` arr server. Resolved at use-time, so edits in Requests propagate.
**Resolution:** on each poll the host resolves the chosen connection (own or linked) into concrete `{base_url, api_key}` and passes them in `PollChangesRequest.connection` (§5). The plugin is credential-source-agnostic — it only ever sees resolved values, fresh each poll, so a change to a reused Requests credential propagates immediately with no re-push. This "reuse or own" behaviour is therefore free for every future provider. (Resolving per-call rather than caching in plugin config also means no decrypted secret is ever stored at rest in the plugin.)
**Decoupling from Requests:** the link is **soft and optional** — `SET NULL` / surfaced-as-"needs attention", never cascading. Requests has no knowledge of Autoscan. Autoscan and Requests are peers; Autoscan merely *offers* to borrow a Requests connection.
## 9. Host: Autoscan category & data model
New top-level **Autoscan** admin section (its own nav entry, not a Requests tab). Manages **Connections** (§8) and **Sources**.
A **source** corresponds to one configured `scan_source.v1` capability instance and carries:
- **Host-owned per-source config:** path rewrites (`autoscan_sources.path_rewrites`, edited in the Autoscan Sources UI) and any provider-specific settings. _(Revised post-merge: rewrites moved host-side; the plugin has no config.)_
- **Host-owned per-source state:** the chosen connection, enabled on/off, poll interval (per-source, not one global timer), the opaque marker, and last-run status/error.
**Install flow + source auto-discovery (resolves §14 risk #1).** A scan-source plugin (Sonarr/Radarr, etc.) is installed through the **normal `/admin/plugins`** page — there is no separate install path; the installer doesn't special-case the capability type. Source rows are then never created by hand: the host enumerates every installed `scan_source.v1` capability (via the plugin installation store) and *seeds a disabled, connection-less row* per capability (`INSERT … ON CONFLICT (installation_id, capability_id) DO NOTHING`, never disturbing existing config). Discovery runs **both on each poll cycle AND when the Autoscan sources list is viewed** (so a freshly-installed plugin appears in Autoscan immediately, even when autoscan is disabled and nothing is polling). The operator then binds a connection and enables it. Consequently **`connection_id` is nullable**: a freshly-discovered source has none, and the engine skips any enabled source with no connection (recording a "no connection bound" status). The admin API rejects enabling a source that has no effective connection.
**Per-source interval** is honoured as a floor: the global poll task runs at `default_poll_interval_seconds`, and within a cycle the engine skips a source whose `last_run_at` is newer than its own `poll_interval_seconds` (falling back to the default). A source therefore polls *at most* every N seconds, never more often than the global cadence.
### Schema changes (host)
Rework the PR #43 tables; **no FK into `request_integrations`**:
- `autoscan_connections` — `id`, `name`, `kind` (e.g. `sonarr`/`radarr`), and *either* own credentials (`base_url`, `api_key_ref` encrypted) *or* a nullable soft link `request_integration_id` (`ON DELETE SET NULL`).
- `autoscan_sources` — `id`, `(installation_id, capability_id)` (unique; the discovered identity), **nullable** `connection_id → autoscan_connections` (`ON DELETE RESTRICT`; null until the operator binds one), `enabled`, `poll_interval_seconds` (nullable; null = use default), `marker` (opaque text, nullable), `last_run_at`, `last_error`. Creation-time validity for connections (own creds OR a link) is enforced at the application layer (the DB CHECK would conflict with `ON DELETE SET NULL` on the reuse link).
- `autoscan_settings` (optional) — retain a global enable + default interval; per-source interval overrides it.
A migration supersedes PR #43's `171_autoscan` schema (which is not yet upstream, so this is a forward redefinition, not a production migration of live data on `origin/main`).
## 10. Plugin: Sonarr/Radarr scan-source
A new installable repo structured like `silo-plugin-tmdb` (standalone Go module, `manifest.json` declaring `scan_source.v1`, gRPC `main.go`, depends on the new tagged `silo-plugin-sdk`). Receives resolved arr `{base_url, api_key}` as config. Implements `PollChanges`:
1. Treat `marker` as the "since" timestamp (empty ⇒ now).
2. Call arr `/api/v3/history/since`, capped by a bounded window (24h max-lookback floor + overlap buffer — relocated from PR #43) so a long outage cannot pull an oversized response.
3. Extract paths: **imports** (`downloadFolderImported.importedPath`) and **renames** (`episodeFileRenamed`/`movieFileRenamed`: both new `path` and old `sourcePath`). Deletes ignored (upgrade-deletes are covered by the paired import; standalone deletes carry no path).
4. Return `{source_paths = the RAW arr paths, next_marker = newest history timestamp}`. The plugin does NOT rewrite — the host normalizes separators and applies the source's rewrite rules. _(Revised post-merge.)_
**Rewrite config + "Sync from arr":** the rewrite list and the suffix-match suggester (which matches arr root folders to Silo media folders) live in the plugin. The suggester needs Silo's media-folder list, obtained via the existing host library-listing service (`internal/pluginhost` library lister) exposed to plugins. The relocated logic is PR #43's `suggest.go`/`suggest_deps.go`/`rewrite.go`/`history.go`.
## 11. Backward compatibility (hard requirement)
Adding `scan_source.v1` **must not break existing plugins.** It doesn't, by construction:
- Capabilities are independent opt-in declarations. Each host subsystem **filters for its own type and ignores the rest** (verified: `metadata_provider.v1`, `scheduled_task.v1`, `event_consumer.v1`, … each use `if type != mine { skip }`). There is **no global allowlist that rejects unknown capability types.**
- **Additive-only rule:** the SDK change *adds* new proto messages/service + a new capability string. It never modifies the shape of an existing capability. (Mutating an existing capability is the only thing that would break plugins; we forbid it.)
- **No forced upgrades:** existing plugin repos stay pinned to their current SDK version and build unchanged. Only a plugin that *wants* to be a scan source adopts the new SDK version.
## 12. Build order & repo decomposition
One shared contract, three repos (+ catalog), in dependency order:
1. **`silo-plugin-sdk`** (long pole): define `scan_source.v1` (proto + generated code + manifest helper + runtime server scaffold), tag a new version (e.g. `v0.5.0`). The tagged release is a hard serialization point — host and plugin cannot build against the contract until it exists.
2. **silo-server (host)** — bump SDK dep; build AutoscanStore (generalized), the `scan_source.v1` caller (timer → `PollChanges`), connections + resolution, the Autoscan category UI, retire the in-process arr autoscan, add the reworked migration. *Parallelizable with (3) once (1) is tagged.*
3. **arr plugin (new repo)** — depends on SDK `v0.5.0`; implement `PollChanges`, rewrites, "Sync from arr", bounded window.
4. **`silo-plugins`** (catalog) — add the entry so the arr plugin is installable.
Each repo gets its **own implementation plan** (separate `writing-plans` pass); this single spec defines the shared contract and per-side responsibilities they all build against.
**Local logistics:** only `silo-server` and `silo-plugin-tmdb` are currently checked out. Implementation will require `silo-plugin-sdk` and a new arr-plugin repo (and later `silo-plugins`) available locally.
## 13. Salvage map (from closed PR #43)
| PR #43 code | Destination |
|---|---|
| arr `/history` client, imports + renames, bounded window (`history.go`) | arr plugin |
| path rewrite apply (`rewrite.go`) + "Sync from arr" suggester (`suggest.go`, `suggest_deps.go`) | arr plugin |
| `last_poll_at` cursor | host, generalized to opaque per-source `marker` |
| resolve + dedupe + suppress + enqueue (`service.go`, `dedupe.go`, `suppress.go`) | host AutoscanStore (generic) |
| `171_autoscan` schema (FK → `request_integrations`) | host, reworked: `autoscan_connections` + `autoscan_sources`, no requests FK |
| Autoscan admin tab under Requests (`AdminRequests.tsx`) | host, as its own Autoscan category |
## 14. Risks & open questions
1. **Plugin multiplicity for multiple arr servers.** Does the runtime support multiple configured instances of one plugin (one per arr server), or is config per-installation only? If the latter, "two Sonarrs + one Radarr" implies either multiple installations or a plugin that manages several connections internally. *Resolve before the host/plugin plans by reading `internal/plugins` installation + capability-instance handling.*
2. **Plugin config UI richness.** The rewrite table + a "Sync from arr" action are richer than typical key-value plugin config. Verify the plugin config-schema/admin-config UI can host a structured list and an action button; if not, design a minimal affordance (e.g. suggester runs on first configure, or a dedicated plugin HTTP route via `http_routes.v1`).
3. **Library-folder list exposure to plugins.** Confirm the `internal/pluginhost` library lister is reachable by a `scan_source` plugin for the suggester. If not, add a host service method.
4. **Marker persistence vs at-least-once.** Ensure marker write happens only after successful enqueue, matching PR #43's "don't advance on failure" guarantee.
5. **Closed PR disposition.** The in-process autoscan on local `main` (34 commits) should be removed/replaced rather than shipped; sequence the host plan to retire it cleanly.
## 15. Testing strategy
- **SDK:** proto compiles; manifest helper round-trips `scan_source.v1`; runtime scaffold serves `PollChanges`.
- **Host engine:** table-driven tests for resolve→suppress→enqueue→marker (port PR #43's `service_test.go`, generalized to a fake `ScanSource` client): dedupe, distinct-paths-same-folder, disabled no-op, poll failure keeps marker, enqueue failure releases claims + keeps marker, opaque-marker pass-through. Run in the libvips-equipped environment (host handler tests link libvips via CGO — see workspace note).
- **Connections:** resolution of own vs linked; linked falls back gracefully when the Requests row is deleted (`SET NULL`).
- **arr plugin:** history parsing (imports + renames, ignores unrelated events), rewrite application, bounded window, first-run "from now"; suffix-match suggester cases (unique/ambiguous/covered/no-op/normalization) ported from PR #43.
- **Compat:** an existing-style plugin manifest with no `scan_source.v1` is unaffected by host scan-source dispatch.
## 16. Out of scope / future
- inotify and Ceph-notify scan-source plugins (the contract accommodates them via the buffer-and-flush pattern of §6).
- A push/event delivery mode, if pull latency ever proves insufficient.
- A `TestConnection` RPC on the capability.
- Migrating Requests onto a shared connection abstraction (today's reuse is a one-directional soft link only).
docs/superpowers/specs/2026-06-02-autoscan-rewrite-sync-design.md
+173
View File
@@ -0,0 +1,173 @@
# Autoscan: Sync Path Rewrites from Arr Root Folders
**Status:** Design approved, ready for implementation plan
**Date:** 2026-06-02
**Area:** `internal/autoscan`, `internal/api`, `web/src/pages/AdminRequests.tsx`
**Extends:** the autoscan feature (`docs/superpowers/specs/2026-06-02-autoscan-arr-polling-design.md`)
## Summary
Autoscan resolves an imported file path against Silo's media folders, but
Radarr/Sonarr often report files under paths that differ from how Silo mounts
the same content — so operators must hand-author per-root `path_rewrites`. For a
multi-instance, many-root deployment this can be dozens of rows.
This adds a **"Sync rewrites from arr"** convenience: for one instance, query its
root folders, match each root to a Silo media folder by **shared trailing path
segments**, and present the resulting `{from, to}` rewrites as a **preview** the
admin reviews and commits. Manual rewrites remain the source of truth; sync only
*fills the editor* — nothing persists until the admin saves.
The matcher is purely structural (no assumptions about any particular path
layout) so it helps any deployment whose arr and Silo share folder names at some
depth, and degrades gracefully (no shared names → "unmatched" → manual entry)
when they do not.
## Goals
- One-click suggestion of `path_rewrites` for an instance from its arr root folders.
- Correct disambiguation when multiple Silo folders share a leaf name.
- Preview-then-commit: the admin sees proposed/unmatched/ambiguous and saves
through the existing per-source save. Manual rows are never removed or overwritten.
- Deployment-agnostic: no hard-coded paths; works for any structure; surfaces
match confidence so weak matches can be rejected.
## Non-Goals
- No automatic/background writing of rewrites — the admin always commits.
- No new persistence path — apply is the existing `PUT /sources/{id}`.
- No case-insensitive matching (case-sensitive segments; correct on Linux). A
toggle can be added later if needed.
- No "sync all instances at once" — per-instance, matching the editor UI.
## Design
### 1. Matching function (core, pure)
A single side-effect-free function in `internal/autoscan`:
```go
func suggestRewrites(arrRoots []string, siloFolderPaths []string, existing []PathRewrite) RewriteSuggestions
```
```go
type RewriteSuggestions struct {
Proposed []ProposedRewrite `json:"proposed"`
Unmatched []string `json:"unmatched"`
Ambiguous []AmbiguousRoot `json:"ambiguous"`
Covered []string `json:"covered"`
}
type ProposedRewrite struct {
From string `json:"from"` // arr root
To string `json:"to"` // matched Silo folder path
MatchDepth int `json:"match_depth"` // trailing segments shared (confidence)
}
type AmbiguousRoot struct {
Root string `json:"root"`
Candidates []string `json:"candidates"`
}
```
**Normalization** (applied to every path before comparison): backslashes → `/`
(reuse `normalizeSeparators`), strip a trailing `/`, collapse duplicate slashes.
This makes any path format (POSIX, Windows, single- or many-root) comparable.
**Per arr root:**
1. **Covered-first:** if any `existing` rewrite's `From` already matches the root
(exact, or root starts with `From + "/"`, using the same boundary rule as
`applyRewrites`), classify as **Covered** and skip — so sync never re-proposes
what a manual or broad rule already handles.
2. Split both the root and each Silo folder path into segments; compute the
**longest common trailing-segment suffix** (full-segment equality, so `4ktv7`
does not match `4ktv70`).
3. Among Silo folders, take the maximum suffix length `> 0` and the folders that
achieve it:
- exactly one → **Proposed** `{From: root, To: folder, MatchDepth: n}`
- more than one → **Ambiguous** `{root, candidates}`
- zero → **Unmatched**
No deployment constants appear in the function; it operates only on the string
slices it is given.
### 2. Preview endpoint & service
**Endpoint:** `GET /api/v1/admin/autoscan/sources/{id}/rewrite-suggestions` →
`RewriteSuggestions` JSON (admin-gated, like the other autoscan routes).
**Service method** `Service.SuggestRewrites(ctx, integrationID) (RewriteSuggestions, error)`:
1. `Repository.GetSource(id)` — a new single-source variant of the existing join
that returns `kind`/`base_url`/`api_key_ref`/`path_rewrites` even when the
source is disabled. Not found → `ErrIntegrationNotFound`.
2. Decrypt the API key via the existing `SecretResolver`.
3. `RootFolderClient.RootFolders(ctx, baseURL, apiKey)` → arr root paths.
4. `FolderLister.ListFolderPaths(ctx)` → all Silo media-folder paths.
5. Return `suggestRewrites(arrRoots, siloPaths, source.PathRewrites)`.
**Two new narrow `Service` dependencies** (injected like `history`/`resolver`):
- `RootFolderClient.RootFolders(ctx, baseURL, apiKey string) ([]string, error)` —
an `arrclient`-backed impl reusing `arrclient.ListRootFolders` (returns paths only).
- `FolderLister.ListFolderPaths(ctx) ([]string, error)` — a thin adapter over
`catalog.FolderRepository.List()` flattening each folder's `Paths`.
**Wiring:** both constructed where the autoscan handler is built in `router.go`
(`deps.FolderRepo` already in scope; the arr client is trivial). The handler's
service interface gains `SuggestRewrites`.
**Errors:** instance not found → 404; arr unreachable / bad key → 502 with the
arr error surfaced; Silo folder-list failure → 500.
No new write path — the frontend merges `proposed` into the editor and the admin
saves via the existing `PUT /sources/{id}`.
### 3. Frontend (preview UI)
In `AutoscanSourceEditor` (`web/src/pages/AdminRequests.tsx`), add a **"Sync from
arr"** button beside Save. On click, a new hook
(`useAutoscanRewriteSuggestions`) calls the endpoint and opens a preview panel:
- **Proposed** — each row: `From → To` + a **confidence badge** from `match_depth`
(e.g. "3 segments" strong, "1 segment — weak"), with a checkbox (default checked).
- **Unmatched** — info-only list of arr roots with no Silo match ("add manually").
- **Ambiguous** — arr root + candidate Silo paths (info-only; admin adds a manual row).
**"Add selected to rewrites"** merges the checked proposals into the editor's
existing rewrite rows, **deduped by `from`, never removing manual rows**. The
admin then clicks the normal **Save**. New types in `web/src/api/types.ts`
(`AutoscanRewriteSuggestions`, `ProposedRewrite`, `AmbiguousRoot`) and hook in
`web/src/hooks/queries/useAutoscan.ts`. Reuse existing
`Button`/`Badge`/`Dialog`/`Field` primitives.
### 4. Testing
- **`suggestRewrites`** — table-driven, fixtures spanning layouts: multi-segment
match, single-segment unique, longest-suffix disambiguation (`Anime/Subs` vs
`Anime2/Subs`), unmatched (no shared segment), already-covered via a broad
prefix rule, an ambiguous tie, plus **unlike-this-deployment** cases
(`/data/Movies` arr vs `/library/Films` Silo → unmatched; `/srv/tv/Show` vs
`/tank/television/Show` → leaf match) and normalization (trailing slash,
backslashes, duplicate slashes).
- **`Service.SuggestRewrites`** — fakes for source store / `RootFolderClient` /
`FolderLister` / secrets: asserts proposed/unmatched/ambiguous; arr-unreachable
surfaces the error.
- **Handler** — returns the JSON; 404 for missing instance; 502 when the arr is
unreachable.
- **Frontend** — the button hits the endpoint; the preview renders all three
buckets with confidence badges; "Add selected" merges deduped rows; Save
persists via the existing mutation.
## Risks & open considerations
- **Weak (1-segment) matches** can be coincidental across very different layouts;
mitigated by surfacing `match_depth` so the admin rejects them before saving.
- **Case-sensitivity** means case-mismatched mounts produce Unmatched rather than
a wrong auto-match — a deliberate safe default; revisit only if requested.
- **Arr reachability** is required for sync (unlike the rest of autoscan which
polls on a schedule); a 502 with the arr error tells the admin to check the
instance, and manual entry still works.
## References
- Reused: `internal/requests/arrclient.ListRootFolders`,
`internal/catalog/FolderRepository.List`, `internal/autoscan` (`PathRewrite`,
`normalizeSeparators`, `Repository`, `Service`, the existing
`PUT /autoscan/sources/{id}` save).
docs/superpowers/specs/2026-06-05-autoscan-source-labels-design.md
+161
View File
@@ -0,0 +1,161 @@
# Autoscan source labels — design
**Date:** 2026-06-05
**Status:** Approved, pending implementation plan
**Scope:** Admin UI labeling of autoscan scan sources, generic across plugins, plus an operator-editable per-source label.
Commands assume the repository root is the cwd.
## Problem
Autoscan sources are labeled in the admin UI (Sources tab and Activity tab) by their
`capability_id` plus `plugin #<installation_id>`. The `arr` scan-source plugin fans out
one source per connection (Sonarr, Sonarr4k, Radarr, Radarr4k) under a single generic
`arr` capability, so every row rendered an identical "arr (plugin #4)" label. A prior
change made the Sources/Activity labels lead with the bound connection name, which fixes
`arr`, but the fallback for any other plugin is still the cryptic `capability_id` +
`plugin #N`.
Two gaps remain:
1. **Other plugins look cryptic.** A connectionless plugin (e.g. cephfs) or any future
plugin still falls back to the raw `capability_id`, ignoring the human-friendly
`display_name` the plugin already declares in its manifest.
2. **No operator control.** An operator cannot name a source themselves. This is the only
fully-general answer for plugin shapes the automatic chain cannot disambiguate (e.g. a
single plugin installation creating several connectionless sources distinguished only
by their `source_config`).
## Goals
- Generic, plugin-agnostic labels driven by metadata that already exists
(manifest `display_name`, connection `name`), so future plugins look good with **no
host code per plugin**.
- An optional operator-editable label per source that overrides the automatic chain.
- A single shared label-resolution helper, consumed by both the Sources and Activity
panels, so the logic is not duplicated.
## Non-goals
- Plugin-declared label templates in the manifest/SDK (the "option C" from brainstorming).
Deferred: it requires a cross-repo SDK/manifest contract change. The chain below leaves a
clean seam for it.
- Backend-side label resolution returned on API payloads. The resolution stays in the
frontend helper for now; promote it to the backend only if the mobile clients need the
same labels.
## Resolution chain
A source's display label resolves through four rungs, most-specific first:
```
operator label -> connection name -> manifest display_name -> capability_id
```
- **operator label** — the optional per-source text the admin sets (this design's new field).
- **connection name** — the operator-named `autoscan_connections.name` bound to the source.
- **manifest display_name** — from the plugin manifest, surfaced by the
`scan-source-plugins` endpoint (`AutoscanAvailableSource.display_name`).
- **capability_id** — last-resort raw identifier; never worse than today.
The chosen rung is the **title**. The remaining identity (plugin display name / capability +
`plugin #<installation_id>`) renders as a **subtitle**, so no information is lost when a
higher rung wins. Example: operator label `4K Movies` as title, `Radarr4k · plugin #4` as
subtitle.
### Per-plugin behavior
| Source shape | Title | Subtitle |
| --- | --- | --- |
| arr, connection bound, no operator label | `Radarr4k` (connection) | `arr · plugin #4` |
| arr, operator label set | `4K Movies` (operator) | `Radarr4k · plugin #4` |
| cephfs, no connection | `CephFS Watcher` (manifest display_name) | `plugin #5` |
| future plugin, nothing declared | `capability_id` | `plugin #N` |
| Activity event whose source was deleted (`source_id` null) | manifest display_name or `capability_id` from the event's stored fields | `plugin #N` |
## Components
### Shared helper — `web/src/lib/autoscanLabels.ts` (new)
Pure, React-free, unit-tested.
- `composeSourceLabel({ operatorLabel, connectionName, displayName, capabilityId, installationId })`
`=> { name: string; detail: string }` — implements the four-rung chain and the
subtitle composition.
- `pluginDisplayNameKey(installationId, capabilityId) => string` — stable map key.
- `buildPluginDisplayNames(available: AutoscanAvailableSource[]) => Map<string, string>` —
builds the `installation_id + capability_id -> display_name` lookup both panels use.
Future seam: `composeSourceLabel` may later accept an optional `configLabel` rung (for the
deferred plugin-template option) without changing either panel's call shape.
### Migration — `migrations/174_autoscan_source_label.{up,down}.sql` (new)
```sql
-- up
ALTER TABLE public.autoscan_sources ADD COLUMN label text NOT NULL DEFAULT '';
-- down
ALTER TABLE public.autoscan_sources DROP COLUMN label;
```
`label` empty string = unset; the resolution chain falls through to the automatic rungs.
174 is a fresh, unused migration number (no collision with the reused-172 history).
### Backend (Go)
- `internal/autoscan/types.go` — add `Label string` to the `Source` struct and the source
input type.
- `internal/autoscan/repository.go` — add `label` to `sourceColumns`, `scanSource`, and the
create/update statements.
- `internal/api/handlers/autoscan.go` — accept `label` on source update; include it in the
`autoscanStatusSource` and source responses. Trim whitespace and cap length at 120
characters server-side.
### Frontend
- `web/src/api/types.ts` — add `label` to `AutoscanSource` and `AutoscanSourceInput`.
- `web/src/pages/admin/autoscan/SourcesPanel.tsx` — replace the inline `sourceIdentity`
block with a call to `composeSourceLabel`. Add an optional "Label" text input to the
source row, saved on blur via the existing `fullBody` PATCH path (mirrors the interval
field's `handleIntervalBlur` auto-save). Build the display-name map from the existing
`useAvailableScanSources()` query.
- `web/src/pages/admin/autoscan/ActivityPanel.tsx` — add `useAvailableScanSources()`, build
the display-name map, map `source_id -> source` to obtain the operator label and
connection, and route `pollSourceName` / `scanSourceName` through `composeSourceLabel`,
returning `.name`. Preserve the existing `"Autoscan"` fallback for scans with no
capability.
## Edit UX
Inline per-row, reusing the existing on-blur auto-save pattern (no modal, no new
interaction model). The label input is blank by default with placeholder text indicating
it is optional. Saving sends the full row state through the same source-update mutation
used for connection/interval/rewrites.
## Error handling and edge cases
- **Deleted source on an Activity event** — `autoscan_events.source_id` is
`ON DELETE SET NULL`; with a null `source_id` there is no operator label or connection,
so the chain resolves from the event row's own stored `capability_id` / `installation_id`
(plus manifest display_name if the plugin is still installed).
- **Uninstalled plugin** — `display_name` does not resolve; the chain falls to
`capability_id`.
- **Bound connection** — `autoscan_sources.connection_id` is `ON DELETE RESTRICT`, so a
bound connection always resolves to a name.
- **Length / whitespace** — the operator label is trimmed and capped at 120 characters
server-side; an all-whitespace label normalizes to empty (unset).
## Testing
- `web/src/lib/autoscanLabels.test.ts` — covers each rung of the chain: operator label
wins, connection name wins, manifest display_name wins, capability fallback, and the
subtitle composition.
- Extend the existing autoscan repository/handler Go tests to cover the `label` column
round-trip (create/update/read) and the server-side trim + length cap.
## Scope summary
One migration pair, three Go files, four frontend files (one new). No SDK changes, no
multi-repo coordination. Operator-editable labels (this design) and the automatic chain
share the same helper, so the deferred plugin-template option layers on later with no
rework.
docs/superpowers/specs/2026-06-06-marker-sources-and-contribution-design.md
+472
View File
@@ -0,0 +1,472 @@
# Marker sources & TheIntroDB contribution — design
**Date:** 2026-06-06
**Status:** Draft (rev 2), pending review
**Scope:** Review the current TheIntroDB marker integration, formalize a multi-source
marker-provider model (query-all / best-wins), and add the ability to contribute Silo's own
markers back to TheIntroDB. This round designs the **server-side API and data model only** —
web-admin and mobile surfaces are deferred to a UI-only follow-up. No client (Android/Apple)
changes required.
Commands assume the repository root is the cwd.
**Rev 2 changes:** contribution is configured **per provider** (off by default); all new web
UI is **deferred** and the backend API is specified in full instead.
## Decisions locked for this design
These were chosen up front and constrain everything below:
1. **Deliverable now:** design/plan only. No source changes land from this document; it is
the blueprint a follow-up implementation plan expands.
2. **What we contribute:** *both* local auto-detected markers *and* manual corrections. Local
detection only produces **intro markers for episodes**, so credits/recap/preview and all
movies can only be contributed via manual correction — this asymmetry drives the design.
3. **Account model:** a single server-wide TheIntroDB account, reusing the existing
`introdb.api_key` server setting. Per-user accounts are a noted future seam, not built now.
4. **Multi-source dispatch:** query all enabled online sources and keep the highest-quality
result per segment ("best wins"). This makes *real per-segment confidence* (gap #2 below)
a hard prerequisite, not an optional nicety.
5. **Contribution is configured per provider, off by default.** Every contribution-capable
provider has its own enable switch and submit settings, so when more providers exist an
operator chooses *which* ones to submit to. Both the master contribute switch and the
auto-submit-local-detections switch default to off; a single setting flips each on.
6. **API-first, UI deferred.** This round fully specifies the backend (manual marker
read/write/clear, contribution trigger + status, per-provider provider config, key
validation and stats) so any future surface — web admin or mobile — builds on a complete
contract. No new web UI is designed here beyond noting where it will eventually attach.
## Problem
Silo currently detects intro/recap/credits/preview markers from two places — a local
chromaprint/chapter analyzer (`internal/intromarkers`) and the online TheIntroDB provider
(`internal/markers/introdb`) — reconciled by a source-priority ladder at write time. The
online side is cleanly abstracted behind a `markers.Provider` interface and a
`markers.Registry`, which is good, but three things are missing:
1. The current TheIntroDB provider has **correctness gaps** measured against the real v3 API
contract (TVDB lookups dropped, real confidence discarded — see "Review findings").
2. There is **no path to contribute** Silo's markers back to TheIntroDB, even though the v3
API exposes a submission endpoint and the package was deliberately built read-only.
3. The provider abstraction is **fetch-only and first-hit**, so it does not yet support
"several online sources, keep the best answer" or "this source can also accept
contributions, and I choose which sources to submit to."
## Review findings — current implementation
### What is already right (keep it)
- **Clean source abstraction.** `markers.Provider` (`ID()`, `FetchMarkers()`) + `markers.Registry`
with `FetchFirstHit` (`internal/markers/types.go`). TheIntroDB is *not* hardcoded; a second
provider is one `Register()` call (`cmd/silo/main.go`).
- **Multi-source-aware schema.** `media_files` carries per-segment provenance for all four
kinds: `{intro,credits,recap,preview}_markers_{source,provider,confidence,algorithm,detected_at}`
(migrations 108/109/137).
- **Write-time reconciliation across source classes.** `CanWriteMarker`
(`internal/markers/write.go`) + `MarkerSourcePriority` (`internal/models/marker_source.go`):
`manual(4) > online/plugin(3) > s3(2) > scanner(1)`, confidence breaks ties. Local detection
uses its own priority-gated path (`intromarkers.Repository.PatchIntroMarker`).
- **Solid HTTP hygiene** in `internal/markers/introdb/client.go`: conservative rate limiter
(2 req/s, burst 5; documented limit is 30 req/10s), retry with exponential backoff,
`Retry-After` handling, 24h TTL cache with negative caching, hot-reloadable API key over
Redis pub/sub.
### Gaps (measured against TheIntroDB's own v3 client + Jellyfin plugin)
| # | Gap | Evidence | Impact |
|---|-----|----------|--------|
| 1 | **TVDB lookups silently dropped.** Provider reads only TMDB+IMDB; client can't send `tvdb_id`. | `internal/markers/introdb/provider.go` (reads `ExternalIDKeyTMDB`/`IMDB` only); `client.go` `FetchEpisode/FetchMovie` build `tmdb_id` else `imdb_id`. | Titles matched only by TVDB (very common for anime / TheTVDB-first libraries) get **zero** markers, even though the resolver already populates `TvdbID` and the v3 API + both official clients support `tvdb_id`. Correctness bug. |
| 2 | **Real confidence + submission_count discarded; hardcoded 0.9.** | `internal/markers/introdb/types.go` `segmentTimestamps` has only `start_ms`/`end_ms`; `pickMarker` sets `Confidence: 0.9`. | The `/media` response carries per-segment `confidence` and `submission_count`. Without them, "best wins" merging can't rank sources, and write-time confidence tie-breaks are meaningless. Hard blocker for the chosen dispatch model. |
| 3 | **Naive multi-candidate selection.** `pickMarker` takes the *first* usable entry. | `internal/markers/introdb/provider.go`. | When several candidates remain (e.g. no duration match), first-usable may not be the most-submitted/highest-confidence. Minor today (duration narrows the set), but compounds once we rank by confidence. |
| 4 | **No contribution path** (by design). | `internal/markers/introdb/types.go` package doc: "Submissions are intentionally not supported." | The whole point of this effort. |
| 5 | **Provider is fetch-only.** No submit capability in the interface. | `markers.Provider` in `internal/markers/types.go`. | Contribution needs an optional `Submitter` capability so non-contributing sources don't have to implement it, and so submission is per provider. |
| 6 | **No key validation / contribution stats.** | `IntroDBCredentialCard` in `web/src/pages/admin-settings/IntegrationsSettings.tsx` only sets the key. | v3 exposes `GET /v3/user/stats` to validate a key and show contribution totals/streaks. Useful "connect & verify" probe, currently absent from the API. |
| 7 | **Single collapsed confidence in the write payload.** `BuildUpdatePayload` promotes one max confidence across all segments. | `internal/markers/write.go`. | The DB has *per-segment* confidence columns, but a merged result (intro from source A, credits from source B) can't populate them independently until per-segment confidence is threaded through the payload. Coupled to gap #2. |
## Authoritative API contract (TheIntroDB v3)
Confirmed from TheIntroDB's own `theintrodb` npm client (`src/funcs.ts`, `src/types.ts`) and
the official Jellyfin plugin (`TheIntroDbClient.cs`). Base URL: `https://api.theintrodb.org/v3`.
### `GET /media` — read (public; optional key)
Query: one of `tmdb_id` / `tvdb_id` / `imdb_id` (preference order tmdb → tvdb → imdb),
plus `season` + `episode` together for TV, plus optional `duration_ms` (selects the closest
release version). Optional `Authorization: Bearer <key>` additionally includes that user's
*pending* submissions.
Response — each segment kind is an array of:
```jsonc
{ "start_ms": 30000|null, "end_ms": 90000|null, "confidence": 0.93|null, "submission_count": 7|null }
```
`start_ms:null` = "from the beginning" (intro/recap); `end_ms:null` = "to the end"
(credits/preview). Arrays may hold multiple entries (multiple release versions) — pick by
duration match, then by `submission_count`/`confidence`.
### `POST /submit` — contribute (requires key)
`Authorization: Bearer <user key>` required; submissions are credited to that account.
Body (one segment per call):
```jsonc
{
"tmdb_id": 1234, // required
"imdb_id": "tt0903747", // optional
"type": "tv" | "movie",
"segment": "intro" | "recap" | "credits" | "preview",
"season": 1, // TV only (number, or "1,2,3" multi-select)
"episode": 1, // TV only
"video_duration_ms": 2760000, // optional, strongly recommended (release-version match)
"start_ms": 0|null, // intro/recap may be null (= beginning)
"end_ms": 90000|null // credits/preview may be null (= end)
}
```
Response: `{ "submissions": [ { "id": uuid, "status": "pending"|"accepted"|"rejected",
"weight": number, ... } ] }`. Submissions enter as `pending`; the backend weight-averages
timestamps across submissions and moderates to accepted/rejected.
Validation rules to mirror client-side: intro/recap require an **end**; credits/preview
require a **start**; TV requires season+episode; movies omit them; max timestamp 6h
(21,600,000 ms); `video_duration_ms` is 0 (unknown) or ≥ 300,000 ms.
### `GET /user/stats` — validate key + contribution stats
`Authorization: Bearer <key>`. Returns `{ total, accepted, pending, rejected,
acceptance_rate, current_streak, best_streak, total_time_saved_ms, top_media }`. A 2xx with no
`error` means the key is valid. This is both our key-validation probe and the stats display.
### Rate / usage limits
429 responses carry `X-RateLimit-{Limit,Remaining,Reset}` and `X-UsageLimit-{Limit,Remaining,Reset}`.
Reads count against the rate limit; **submissions count against the usage limit**. The
contribution pipeline must honor `X-UsageLimit-Reset` and back off — reuse the existing
limiter and add usage-limit awareness.
## Target architecture
Three layers, each independently shippable. Layer A is pure correctness and unblocks B and C.
### Layer A — Correctness foundation (gaps #1, #2, #3, #7)
- **TVDB in the client + provider.** Add `tvdb_id` to the query builder in
`internal/markers/introdb/client.go` (`FetchEpisode`/`FetchMovie` gain a `tvdbID` arg,
preference tmdb → tvdb → imdb, cache keys extended). Provider reads
`req.ExternalIDs[markers.ExternalIDKeyTVDB]`. Resolver already supplies it — no resolver
change.
- **Capture confidence + submission_count.** Extend `segmentTimestamps` with
`Confidence *float64` and `SubmissionCount *int`; add `SubmissionCount int` to
`markers.Marker`; `pickMarker` uses the real confidence (fallback to a named default
constant, e.g. `0.9`, only when the field is null) instead of hardcoding.
- **Best-candidate selection.** `pickMarker` ranks candidates by `(submission_count desc,
confidence desc)` after duration filtering, instead of first-usable.
- **Per-segment confidence in the payload.** Replace the single `Confidence *float64` on
`markers.MarkerUpdatePayload` with per-segment confidence (e.g. `IntroConfidence`,
`CreditsConfidence`, …) so a merged multi-source result writes the correct
`{kind}_markers_confidence` per kind. Update `BuildUpdatePayload` and the
`scanner.FileRepository.UpsertMarkers` column mapping accordingly.
### Layer B — Multi-source dispatch ("query all, best wins")
- **`Submitter` capability (optional interface).**
```go
// internal/markers/types.go
type Submitter interface {
Provider
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
// FetchUserStats validates the configured key and returns contribution stats.
FetchUserStats(ctx context.Context) (UserStats, error)
}
```
Only `introdb.Provider` implements it; the contribution service type-asserts
`provider.(markers.Submitter)`. This is also what makes contribution per-provider: the
service only ever submits to registered providers that satisfy `Submitter` *and* are enabled
in `marker_provider_config` (C6).
- **Merge dispatch.** Add `Registry.FetchMerged(ctx, req) (Result, bool, error)` that fans out
to all *fetch-enabled* providers concurrently (bounded), then merges **per segment kind**
keeping the candidate with the highest `(submission_count, confidence)`. `FetchFirstHit`
stays as the trivial single-provider path / fallback. With only TheIntroDB enabled,
`FetchMerged` behaves identically to today — zero behavior change until a second source
exists.
- **Cross-source-class reconciliation is unchanged.** `FetchMerged` reconciles *within* the
online class; `CanWriteMarker` still arbitrates online vs scanner vs manual at write time.
No double logic.
- **Provider enable/order config.** Driven by the `marker_provider_config` table (C6):
`fetch_enabled` selects which providers are queried, `fetch_priority` is the merge
tiebreaker. The metadata `library_provider_chains` DB pattern (`internal/metadata/chain.go`)
is the upgrade path *if* per-library control is ever needed — noted, not built.
- **Future plugin sources (deferred seam).** A `marker_provider.v1` plugin capability would let
third-party sources plug in. The clean seam is a `PluginMarkerProvider` adapter implementing
`markers.Provider` (mirroring `internal/metadata/plugin_provider.go`), so the registry never
changes. Cross-repo (silo-plugin-sdk) — explicitly out of scope here.
### Layer C — Contribution to TheIntroDB
#### C1. Submission client
Add to `internal/markers/introdb/client.go`:
- `SubmitSegment(ctx, SubmissionRequest) (SubmissionResult, error)` → `POST /v3/submit`,
Bearer key **required** (error early if `introdb.api_key` is empty), usage-limit aware.
- `FetchUserStats(ctx) (UserStats, error)` → `GET /v3/user/stats`.
`introdb.Provider` implements `markers.Submitter` by delegating to these.
#### C2. What is eligible to contribute
A marker is contributable iff:
- the target provider is a registered `Submitter` with `contribute_enabled = true` (C6), and
- its segment source ∈ `{scanner, manual}` (**never** `online` — that came *from* an online
provider; contributing it back is circular), and
- the file resolves to a usable external ID (tmdb/tvdb/imdb) via the existing resolver, and
- for **auto** contribution: provider `contribute_auto_local = true`, source = `scanner`,
kind = `intro`, confidence ≥ that provider's `contribute_min_confidence`. (Local detection is
intro-only/episodes-only.)
- for **manual** contribution: any kind, episodes or movies, source = `manual`.
#### C3. Manual marker API (write path; UI deferred)
Markers live on `media_files`, and an item can have several versions, so the manual API is
**file-scoped** for precision, with an item-level convenience resolver. All under the existing
`RequireAdmin` group (`internal/api/middleware/auth.go`). New `internal/api/handlers/admin_markers.go`:
- `GET /admin/files/{fileId}/markers` — current per-segment values **plus full provenance**
(`source`, `provider`, `confidence`, `algorithm`, `detected_at`) for each of
intro/recap/credits/preview. The read side an editor (or a script) needs.
- `PUT /admin/files/{fileId}/markers` — upsert the **manual** layer. Body carries, per segment,
either `{start, end}` (seconds) or `null` to clear that segment's manual value. Writes
`source = manual` (priority 4, beats every other source) through the existing priority-gated
`scanner.FileRepository.UpsertMarkers` path, then notifies live sessions via
`playback.MarkerUpdateNotifier.MarkersUpdated` (mirrors `admin_intro.go`).
- `DELETE /admin/files/{fileId}/markers/{segment}` — explicit single-segment clear.
- `GET|PUT /admin/items/{id}/markers` — item-level convenience that resolves to the primary
file (and reports the per-version set), for callers that don't track file IDs.
**Clear semantics.** Clearing a manual segment nulls that segment's value and provenance;
`markers_source` drops to the next-highest segment still present. A subsequent local
re-detection or online lazy-fetch can repopulate it. (We can't "un-overwrite" a value a manual
edit replaced, so clearing means "remove the manual value," not "restore the previous source.")
**Validation** mirrors the contribution rules so a manual edit can always be contributed:
`end > start`; intro/recap may omit start (= 0); credits/preview may omit end (= file
duration); values within [0, file duration].
A web editor (player-chrome control with a "use current time" helper) attaches to this API
later; its UX is intentionally **out of scope** for this round.
#### C4. Contribution tracking (idempotency + audit)
The audit log that makes submission idempotent and lets us report status. Per-provider by
construction (the `provider` column), so the same marker can be tracked/submitted to several
providers independently.
```sql
-- migrations/<next>_marker_contributions.up.sql (next free number at implementation time)
CREATE TABLE public.marker_contributions (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
provider text NOT NULL, -- 'introdb'
segment_kind text NOT NULL, -- intro|recap|credits|preview
source text NOT NULL, -- what we contributed: 'scanner'|'manual'
submitted_start_ms bigint,
submitted_end_ms bigint,
video_duration_ms bigint,
content_hash text NOT NULL, -- hash(segment_kind,start_ms,end_ms,duration_ms)
submission_id uuid, -- returned by /submit
status text NOT NULL, -- pending|accepted|rejected|error
http_status integer,
error text,
submitted_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (media_file_id, provider, segment_kind, content_hash)
);
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
```
`content_hash` over the *value* gives idempotency: the same times are never resubmitted to the
same provider, but a manual correction (different times → different hash) submits the new
value — which is exactly how TheIntroDB's weighted averaging expects corrections to arrive.
#### C5. Contribution service + triggers
A single `internal/markers/contribute.go` `ContributionService` funnels both triggers:
`resolve enabled submitter providers (C6) → per provider: eligibility (C2) → idempotency check
(hash vs marker_contributions) → Submitter.SubmitMarker → persist result`. Triggers:
- **Manual / on-demand:** `POST /admin/files/{fileId}/contribute` (C7) submits a file's eligible
markers to enabled providers (optionally scoped to one provider/segments).
- **Auto local detections:** a daily `ContributeMarkersTask` (`internal/taskmanager/tasks/`)
that, for each provider with `contribute_enabled && contribute_auto_local`, scans for episode
files with `intro_markers_source = 'scanner'` and `intro_markers_confidence ≥` that provider's
`contribute_min_confidence` lacking a current-hash `marker_contributions` row, and submits
them rate/usage-limited. Runs after the existing 03:30 detection task.
#### C6. Per-provider configuration
Contribution and fetch behavior is **per provider**, so a single global toggle won't do.
Credentials stay in `server_settings` (sensitive handling: `introdb.api_key`); behavioral
config moves to a new table keyed by provider id:
```sql
-- migrations/<next>_marker_provider_config.up.sql
CREATE TABLE public.marker_provider_config (
provider text PRIMARY KEY, -- 'introdb'
fetch_enabled boolean NOT NULL DEFAULT true, -- queried during reads (Layer B)
fetch_priority integer NOT NULL DEFAULT 100, -- merge tiebreaker
contribute_enabled boolean NOT NULL DEFAULT false, -- master submit switch (off)
contribute_auto_local boolean NOT NULL DEFAULT false, -- auto-submit local detections (off)
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
updated_at timestamptz NOT NULL DEFAULT now()
);
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
```
- A provider's submissions require `contribute_enabled = true`. With it on, **manual**
corrections submit on demand; **auto** submission of local detections additionally requires
`contribute_auto_local = true`. Both contribute flags default off — "easily enabled with a
setting," exactly one row per provider.
- This same table carries the Layer B fetch enable/order, so "which providers do we read from"
and "which do we submit to" live in one place. New providers insert a row (contribute off).
- Surfaced and edited through the provider API (C7), so the future admin surface lists
providers generically rather than hard-coding `introdb`.
The `markers.mode` / `markers.lazy_playback` globals and the `introdb.api_key` credential are
unchanged.
#### C7. Admin provider + contribution API (fully fledged)
The complete backend surface, all under `RequireAdmin`, so a UI is purely additive later:
- `GET /admin/markers/providers` — list every registered provider: id, whether it is a
`Submitter`, its `marker_provider_config` row, and (best-effort, cached) live `/user/stats`
for submitter providers that have a key.
- `PUT /admin/markers/providers/{provider}` — update that provider's config row
(`fetch_enabled`, `fetch_priority`, `contribute_enabled`, `contribute_auto_local`,
`contribute_min_confidence`). Validated; unknown provider → 404.
- `POST /admin/markers/providers/{provider}/validate` — validate the configured key and return
fresh `UserStats` (calls `Submitter.FetchUserStats`). Non-submitter → 400.
- `POST /admin/files/{fileId}/contribute` — submit this file's eligible markers (C2) to enabled
submitter providers; optional body `{provider?, segments?[]}` to scope to one provider /
specific kinds. Returns per (provider, segment) `SubmissionResult`. Honors idempotency (C4):
unchanged values are skipped, not resubmitted.
- `GET /admin/files/{fileId}/contributions` — contribution history for the file from
`marker_contributions` (per provider/segment: status, submitted_at, error).
Plus the manual marker endpoints from C3. Together these cover read, edit, submit, and audit
without any web code.
## Data model summary
- **New table `marker_contributions`** (C4) — idempotent submission audit, keyed per
(file, provider, segment, value-hash).
- **New table `marker_provider_config`** (C6) — per-provider fetch + contribute behavior,
seeded with `introdb` (fetch on, contribute off).
- **No new flat server settings.** `introdb.api_key`, `markers.mode`, `markers.lazy_playback`
are unchanged; behavioral provider config lives in `marker_provider_config`, not string keys.
- **No new marker columns.** Per-segment confidence columns already exist; Layer A only changes
how they're *populated* (per-segment instead of collapsed). `submission_count` stays
transient (merge-ranking only) unless a later need appears.
## Client / cross-repo impact
- **No Android/Apple changes required.** This adds no new marker *kinds* and no new
client-facing read contract: markers still flow to clients via the existing `WatchDetail`
fields, the jellycompat `GET /MediaSegments` endpoint, and the `markers_updated` realtime
event. Manual editing and contribution are server-side (admin API); their eventual UI is
deferred web-admin work.
- **Plugin SDK:** untouched. `marker_provider.v1` is a deferred future capability (Layer B
seam), not part of this work.
- A future *per-user* contribution model (each user links their own TheIntroDB account) *would*
be client work (a "Connect TheIntroDB" screen) — explicitly deferred.
## Build sequence
Each phase is shippable on its own; later phases depend only on earlier ones. Every phase is
**backend-only** — no web UI.
1. **Phase 1 — Correctness foundation (Layer A).** TVDB lookups; capture
confidence/submission_count; best-candidate selection; per-segment confidence in the
payload. *Directly answers "are we doing this properly," low risk, no new surface.*
2. **Phase 2 — Multi-source dispatch + provider config (Layer B + C6).** `Submitter` interface;
`FetchMerged` query-all/best-wins; `marker_provider_config` table + repo driving fetch
enable/order. *No behavior change with only TheIntroDB enabled.*
3. **Phase 3 — Submission client + tracking + service (C1, C4, C5 core).** `SubmitSegment` /
`FetchUserStats`; `marker_contributions` migration + repo; `ContributionService` (eligibility,
idempotency, per-provider gating).
4. **Phase 4 — Manual marker API (C3) + provider/contribution API (C7).** File- and item-level
manual read/write/clear; provider list/config/validate; per-file contribute + history. The
full admin contract, no UI.
5. **Phase 5 — Auto-contribution (C5 task).** `ContributeMarkersTask`: per provider, when
`contribute_enabled && contribute_auto_local`, submit episode intro markers above the
provider's `contribute_min_confidence`, usage-limit aware, idempotent.
Deferred (separate UI-only follow-up): web admin surfaces over C3/C6/C7 — a marker editor in
the player chrome and a provider/contribution panel in admin settings.
## Risks & trade-offs
- **Data-quality / pollution.** Auto-submitting chromaprint detections (confidence 0.65–0.90)
risks polluting a provider's data. Mitigation: `contribute_enabled` and `contribute_auto_local`
default **off** per provider, and the default `contribute_min_confidence` of 0.95 admits only
chapter / chapter+silence detections even once enabled. Manual, high-trust corrections are the
intended first contribution. Revisit chromaprint auto-submission after watching acceptance
rates via `/user/stats`.
- **Single-account weighting.** All contributions credit one server account; TheIntroDB's
weighting may treat a single account's mass submissions differently than many users'. Note
the per-user seam; acceptable for v1.
- **Circular contribution.** Never contribute `source = online`. Enforced in C2 eligibility and
worth a unit test.
- **Usage limits.** Bulk auto-submission can hit the submission usage limit; the task must obey
`X-UsageLimit-Reset` and resume next run. Idempotency means an interrupted run safely resumes.
- **"Query all" cost.** `FetchMerged` issues one lookup per fetch-enabled provider; with only
TheIntroDB it's identical to today, and per-provider caching bounds the cost as sources are
added.
- **Movies & non-intro segments** have no local detector, so their only contribution route is
manual correction — acceptable and explicit.
## Open questions for review
1. **Contribute to all enabled providers, or require explicit per-call selection?** The design
defaults `POST /admin/files/{id}/contribute` to "every enabled submitter," with an optional
`provider` scope. Acceptable, or should submission always name its target provider?
2. **Manual-clear behavior.** Clearing a manual segment removes the manual value and lets a
later detection/fetch repopulate it (we can't restore the exact pre-edit value). Is
"remove, then re-detect" the right contract, or do you want a marker history/undo stack?
3. **Per-provider config home.** This design uses a `marker_provider_config` table (clean for N
providers, future per-library override) rather than flat `server_settings` keys. Confirm the
table is the direction you want.
## File-level blueprint
**Backend (Go) — Layer A**
- `internal/markers/introdb/client.go` — *modify*: `tvdb_id` query support; cache-key extension.
- `internal/markers/introdb/types.go` — *modify*: `confidence`/`submission_count` fields.
- `internal/markers/introdb/provider.go` — *modify*: read TVDB; real confidence; best-candidate pick.
- `internal/markers/types.go` — *modify*: per-segment confidence in `Marker`/Result plumbing.
- `internal/markers/write.go` — *modify*: per-segment confidence in `MarkerUpdatePayload` + `BuildUpdatePayload`.
- `internal/scanner/file_repo.go` — *modify*: map per-segment confidence in `UpsertMarkers`.
**Backend (Go) — Layer B + per-provider config**
- `internal/markers/types.go` — *modify*: `Submitter` interface; `SubmissionRequest`/`Result`, `UserStats`; `Registry.FetchMerged`.
- `internal/markers/provider_config.go` — *create*: `marker_provider_config` repo (load/update; drives fetch enable/order + contribute gating).
- `migrations/<next>_marker_provider_config.{up,down}.sql` — *create*.
**Backend (Go) — Layer C**
- `internal/markers/introdb/client.go` — *modify*: `SubmitSegment` (`POST /v3/submit`), `FetchUserStats` (`GET /v3/user/stats`), usage-limit handling.
- `internal/markers/introdb/provider.go` — *modify*: implement `markers.Submitter`; drop the "submissions not supported" note in `types.go`.
- `internal/markers/contribute.go` — *create*: `ContributionService` (eligibility, idempotency, per-provider gating, submit, persist).
- `internal/markers/contribution_repo.go` — *create*: `marker_contributions` CRUD + value-hash check.
- `migrations/<next>_marker_contributions.{up,down}.sql` — *create*.
- `internal/api/handlers/admin_markers.go` — *create*: manual marker read/write/clear (C3) + per-file contribute + history (C7 file endpoints).
- `internal/api/handlers/admin_marker_providers.go` — *create*: provider list/config/validate (C7 provider endpoints).
- `internal/api/router.go` — *modify*: register the new `RequireAdmin` routes.
- `internal/taskmanager/tasks/contribute_markers.go` — *create*: daily auto-contribution task.
- `cmd/silo/main.go` — *modify*: wire `ContributionService`, provider-config repo, the task, and the `Submitter`.
**Frontend (React/TS) — deferred (not this round)**
- Marker editor over `PUT /admin/files/{id}/markers`; provider/contribution panel over the C7
endpoints; `IntroDBCredentialCard` "validate key & show stats". Listed for traceability only.
**Reference (read-only, not in this repo)**
- TheIntroDB v3 contract verified against `github.com/TheIntroDB/theintrodb-npm`
(`src/funcs.ts`, `src/types.ts`, `docs/`) and `github.com/TheIntroDB/jellyfin-plugin`
(`TheIntroDB/Api/*.cs`).
go.mod
+33 -6
View File
@@ -26,29 +26,57 @@ require (
github.com/h2non/bimg v1.1.9
github.com/hashicorp/go-hclog v1.6.3
github.com/joho/godotenv v1.5.1
github.com/mmcdole/gofeed v1.3.0
github.com/oklog/ulid/v2 v2.1.0
github.com/pgvector/pgvector-go v0.3.0
github.com/pressly/goose/v3 v3.27.1
github.com/zishang520/socket.io/v2 v2.5.0
go.n16f.net/thumbhash v1.1.0
golang.org/x/image v0.39.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529
gopkg.in/natefinch/lumberjack.v2 v2.2.1
)
require (
github.com/PuerkitoBio/goquery v1.8.0 // indirect
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/hashicorp/yamux v0.1.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-isatty v0.0.21 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/quic-go/quic-go v0.53.0 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
github.com/zishang520/engine.io-go-parser v1.3.2 // indirect
github.com/zishang520/engine.io/v2 v2.5.0 // indirect
github.com/zishang520/socket.io-go-parser/v2 v2.5.0 // indirect
github.com/zishang520/webtransport-go v0.9.1 // indirect
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
go.uber.org/mock v0.5.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/mod v0.34.0 // indirect
golang.org/x/net v0.53.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
golang.org/x/tools v0.43.0 // indirect
)
require (
github.com/Silo-Server/silo-plugin-sdk v0.4.0
github.com/Silo-Server/silo-plugin-sdk v0.5.0
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
@@ -67,15 +95,14 @@ require (
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/robfig/cron/v3 v3.0.1
github.com/sony/sonyflake v1.3.0
go.opentelemetry.io/otel v1.41.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/sync v0.20.0
golang.org/x/text v0.36.0
golang.org/x/time v0.14.0
google.golang.org/grpc v1.79.3
google.golang.org/grpc v1.80.0
google.golang.org/protobuf v1.36.11
)
go.sum
+97 -23
View File
@@ -1,9 +1,15 @@
entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
github.com/Silo-Server/silo-plugin-sdk v0.4.0 h1:DJkRROQfr/kfwnF5dUdkhmbCso1KDLZc7uK7YfJnaO0=
github.com/Silo-Server/silo-plugin-sdk v0.4.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/Silo-Server/silo-plugin-sdk v0.5.0 h1:0yJpD0RP1AetGsWVCXoPih2ggLjN8hVp8blS+2RkIWU=
github.com/Silo-Server/silo-plugin-sdk v0.5.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
@@ -38,6 +44,7 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -45,8 +52,12 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
@@ -65,8 +76,11 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/h2non/bimg v1.1.9 h1:WH20Nxko9l/HFm4kZCA3Phbgu2cbHvYzxwxn9YROEGg=
@@ -95,8 +109,10 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -112,12 +128,25 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
@@ -127,22 +156,32 @@ github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2
github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.27.1 h1:6uEvcprBybDmW4hcz3gYujhARhye+GoWKhEWyzD5sh4=
github.com/pressly/goose/v3 v3.27.1/go.mod h1:maruOxsPnIG2yHHyo8UqKWXYKFcH7Q76csUV7+7KYoM=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
github.com/quic-go/quic-go v0.53.0 h1:QHX46sISpG2S03dPeZBgVIZp8dGagIaiu2FiVYvpCZI=
github.com/quic-go/quic-go v0.53.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
github.com/sony/sonyflake v1.3.0 h1:tiB4Dlp0lnmKp/h6BLXA14P8Qi+LYS9+0QRpcrKHvg4=
github.com/sony/sonyflake v1.3.0/go.mod h1:LORtCywH/cq10ZbyfhKrHYgAUGH7mOBa76enV9txy/Y=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -161,62 +200,89 @@ github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibgg
github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zishang520/engine.io-go-parser v1.3.2 h1:aEVrhQVhfk99Ct6htNffgHydUBC4dGclO/OXPz5CSy0=
github.com/zishang520/engine.io-go-parser v1.3.2/go.mod h1:fg/R4V7aytYwUTu4lGcPdjenDSXFWLlkDAGewWVOo3o=
github.com/zishang520/engine.io/v2 v2.5.0 h1:0ayZCt51c8lntxG5AWoM2mX40ryZlvRodAULXB1XK/s=
github.com/zishang520/engine.io/v2 v2.5.0/go.mod h1:ohfMsnzOCA9NEklEGiQ5Y9j6cWvzLNeVFaB+Bkn1KcQ=
github.com/zishang520/socket.io-go-parser/v2 v2.5.0 h1:uGKTwcH2qrrs9uwzfCo3ems+qUJZo9beZLK8ZXOwKYo=
github.com/zishang520/socket.io-go-parser/v2 v2.5.0/go.mod h1:GK9GIIs/KQbBKfnxgZJMBYSlsFemB2rL7EFUn0kmTfM=
github.com/zishang520/socket.io/v2 v2.5.0 h1:+KdZLbl4wWVzyI84RG+h21t8OY4ifz/Oo6qwgt9zHl8=
github.com/zishang520/socket.io/v2 v2.5.0/go.mod h1:+GyoPyakXDS6KsW81RAQpDA9+mJBXbcYcQ+Itx2D+rU=
github.com/zishang520/webtransport-go v0.9.1 h1:Y3gqPM8cIDvQILsTyXJ5G9fp2PYqGqLI2z+QXpgboQc=
github.com/zishang520/webtransport-go v0.9.1/go.mod h1:IgNAD6qLe3oWu7MSSkjusRNftpvjYxWjI4LmoH4VEyY=
go.n16f.net/thumbhash v1.1.0 h1:aBEvuAd4yiwzeQ7Sm4BZoHJYbrQ1ewjrmrRlCE79snk=
go.n16f.net/thumbhash v1.1.0/go.mod h1:mo9pP7WtfdV9ojIamGFR/Vc0PaPA2l0CUtmYQf/SweU=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8=
go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y=
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -233,3 +299,11 @@ gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
modernc.org/libc v1.72.1 h1:db1xwJ6u1kE3KHTFTTbe2GCrczHPKzlURP0aDC4NGD0=
modernc.org/libc v1.72.1/go.mod h1:HRMiC/PhPGLIPM7GzAFCbI+oSgE3dhZ8FWftmRrHVlY=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
internal/adminjob/cancel_registry.go
+55
View File
@@ -0,0 +1,55 @@
package adminjob
import (
"context"
"sync"
)
// CancelRegistry bridges HTTP cancellation requests to in-process job contexts.
// Queued jobs are cancelled in the repository; running jobs need their active
// context cancelled so executors stop doing work before the row is finalized.
type CancelRegistry struct {
mu sync.Mutex
cancels map[string]cancelEntry
}
type cancelEntry struct {
cancel context.CancelFunc
token *struct{}
}
func NewCancelRegistry() *CancelRegistry {
return &CancelRegistry{cancels: make(map[string]cancelEntry)}
}
func (r *CancelRegistry) Register(jobID string, cancel context.CancelFunc) func() {
if r == nil || jobID == "" || cancel == nil {
return func() {}
}
token := &struct{}{}
r.mu.Lock()
r.cancels[jobID] = cancelEntry{cancel: cancel, token: token}
r.mu.Unlock()
return func() {
r.mu.Lock()
if entry := r.cancels[jobID]; entry.token == token {
delete(r.cancels, jobID)
}
r.mu.Unlock()
}
}
func (r *CancelRegistry) Cancel(jobID string) bool {
if r == nil || jobID == "" {
return false
}
r.mu.Lock()
cancel := r.cancels[jobID].cancel
r.mu.Unlock()
if cancel == nil {
return false
}
cancel()
return true
}
internal/adminjob/item_refresh.go
+4 -2
View File
@@ -542,17 +542,19 @@ func (e *ItemRefreshExecutor) Execute(ctx context.Context, req ItemRefreshReques
e.publish(cache.EventMetadataUpdated, refreshContentID)
if e.realtimeHub != nil {
if scanResult != nil && (scanResult.New > 0 || scanResult.Updated > 0 || scanResult.Missing > 0 || matched > 0) {
_ = e.realtimeHub.PublishLibraryChanged(ctx, notifications.LibraryChangeEvent{
_ = e.realtimeHub.PublishCatalogLibraryChanged(ctx, notifications.LibraryChangeEvent{
LibraryID: req.ScanFolderID,
Reason: "item_refresh",
New: scanResult.New,
Updated: scanResult.Updated,
Missing: scanResult.Missing,
MatchedFiles: matched,
})
}
_ = e.realtimeHub.PublishMetadataUpdated(ctx, notifications.MetadataUpdateEvent{
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
LibraryID: req.ScanFolderID,
ContentID: refreshContentID,
Change: "metadata_updated",
})
}
internal/adminjob/library_refresh.go
+15 -1
View File
@@ -251,8 +251,9 @@ func (e *LibraryRefreshExecutor) Execute(
if result.RefreshedOK > 0 || result.PipelineOK > 0 {
e.publish(cache.EventMetadataUpdated, strconv.Itoa(req.LibraryID))
if e.realtimeHub != nil {
_ = e.realtimeHub.PublishMetadataUpdated(ctx, notifications.MetadataUpdateEvent{
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
LibraryID: req.LibraryID,
Change: "metadata_updated",
})
}
}
@@ -321,6 +322,7 @@ func (e *LibraryRefreshExecutor) refreshItemsWithIDs(
result.RefreshedFailed++
} else {
result.RefreshedOK++
e.publishCatalogItemChanged(ctx, result.LibraryID, itemResult.contentID)
}
advance("Refreshing items with external IDs")
}
@@ -386,6 +388,7 @@ func (e *LibraryRefreshExecutor) refreshUnmatchedItem(ctx context.Context, libra
if err := e.refresher.RefreshItemForLibrary(ctx, req.RefreshContentID, req.ScanFolderID); err != nil {
return fmt.Errorf("refresh metadata: %w", err)
}
e.publishCatalogItemChanged(ctx, libraryID, req.RefreshContentID)
return nil
}
@@ -399,6 +402,17 @@ func (e *LibraryRefreshExecutor) publish(eventType, payload string) {
})
}
func (e *LibraryRefreshExecutor) publishCatalogItemChanged(ctx context.Context, libraryID int, contentID string) {
if e == nil || e.realtimeHub == nil || contentID == "" {
return
}
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
LibraryID: libraryID,
ContentID: contentID,
Change: "metadata_updated",
})
}
func decodeLibraryRefreshRequest(data json.RawMessage) (LibraryRefreshRequest, error) {
var req LibraryRefreshRequest
if len(data) == 0 {
internal/adminjob/repository.go
+79
View File
@@ -24,11 +24,13 @@ const (
StatusRunning = "running"
StatusCompleted = "completed"
StatusFailed = "failed"
StatusCancelled = "cancelled"
)
var (
ErrJobNotFound = errors.New("admin job not found")
ErrActiveJobConflict = errors.New("admin job already active for type")
ErrJobNotCancellable = errors.New("admin job is not cancellable")
)
type ActiveJobConflictError struct {
@@ -471,6 +473,83 @@ func (r *Repository) Fail(ctx context.Context, id string, input FailJobInput) er
return nil
}
func (r *Repository) Cancel(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error) {
if message == "" {
message = "Admin job cancelled"
}
job, err := scanAdminJob(r.pool.QueryRow(ctx, `
UPDATE admin_jobs
SET status = $2,
message = $3,
error_message = '',
completed_at = NOW(),
heartbeat_at = NOW(),
expires_at = $4,
updated_at = NOW()
WHERE id = $1
AND status IN ($5, $6)
RETURNING `+adminJobColumns,
id,
StatusCancelled,
message,
expiresAt,
StatusQueued,
StatusRunning,
))
if err == nil {
return job, nil
}
if !errors.Is(err, ErrJobNotFound) {
return nil, err
}
existing, lookupErr := r.GetByID(ctx, id)
if lookupErr != nil {
return nil, lookupErr
}
if existing.Status != StatusQueued && existing.Status != StatusRunning {
return nil, ErrJobNotCancellable
}
return nil, ErrJobNotFound
}
func (r *Repository) CancelQueued(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error) {
if message == "" {
message = "Admin job cancelled"
}
job, err := scanAdminJob(r.pool.QueryRow(ctx, `
UPDATE admin_jobs
SET status = $2,
message = $3,
error_message = '',
completed_at = NOW(),
heartbeat_at = NOW(),
expires_at = $4,
updated_at = NOW()
WHERE id = $1
AND status = $5
RETURNING `+adminJobColumns,
id,
StatusCancelled,
message,
expiresAt,
StatusQueued,
))
if err == nil {
return job, nil
}
if !errors.Is(err, ErrJobNotFound) {
return nil, err
}
existing, lookupErr := r.GetByID(ctx, id)
if lookupErr != nil {
return nil, lookupErr
}
if existing.Status != StatusQueued {
return nil, ErrJobNotCancellable
}
return nil, ErrJobNotFound
}
func (r *Repository) RequeueStaleRunning(ctx context.Context, before time.Time) (int, error) {
tag, err := r.pool.Exec(ctx, `
UPDATE admin_jobs
internal/adminjob/runner.go
+32
View File
@@ -57,6 +57,7 @@ type Runner struct {
heartbeatInterval time.Duration
staleAfter time.Duration
retention time.Duration
cancelRegistry *CancelRegistry
stop chan struct{}
stopOnce sync.Once
}
@@ -99,10 +100,17 @@ func NewRunner(
heartbeatInterval: 10 * time.Second,
staleAfter: 2 * time.Minute,
retention: 7 * 24 * time.Hour,
cancelRegistry: NewCancelRegistry(),
stop: make(chan struct{}),
}
}
func (r *Runner) SetCancelRegistry(registry *CancelRegistry) {
if registry != nil {
r.cancelRegistry = registry
}
}
func (r *Runner) Start() {
go func() {
r.requeueStaleJobs()
@@ -329,6 +337,8 @@ func (r *Runner) executeLibraryRefresh(job *models.AdminJob) {
ctx, cancel := context.WithTimeout(context.Background(), libraryRefreshTimeout)
defer cancel()
unregisterCancel := r.cancelRegistry.Register(job.ID, cancel)
defer unregisterCancel()
heartbeatStop := make(chan struct{})
go r.heartbeatLoop(ctx, job.ID, heartbeatStop)
@@ -357,6 +367,10 @@ func (r *Runner) executeLibraryRefresh(job *models.AdminJob) {
r.publishJobByID(ctx, notifications.TypeJobProgress, job.ID)
})
if err != nil {
if errors.Is(ctx.Err(), context.Canceled) {
r.cancelJob(job.ID, current, total, "Library metadata refresh cancelled")
return
}
msg := err.Error()
if ctx.Err() != nil {
msg = fmt.Sprintf("timed out after %s: %s", libraryRefreshTimeout, msg)
@@ -846,3 +860,21 @@ func (r *Runner) failJob(id string, current, total int, message, errorMessage st
}
r.publishJobByID(ctx, notifications.TypeJobFailed, id)
}
func (r *Runner) cancelJob(id string, current, total int, message string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := r.repo.UpdateProgress(ctx, id, current, total, message); err != nil {
slog.Warn("admin jobs: failed to update cancellation progress", "job_id", id, "error", err)
}
job, err := r.repo.Cancel(ctx, id, message, time.Now().UTC().Add(r.retention))
if err != nil {
slog.Warn("admin jobs: failed to mark job cancelled", "job_id", id, "error", err)
return
}
if r.realtimeHub != nil {
if err := r.realtimeHub.PublishJob(ctx, notifications.TypeJobCancelled, job); err != nil {
slog.Warn("admin jobs: failed to publish job cancellation", "job_id", id, "error", err)
}
}
}
internal/api/autoscan_wiring.go
+165
View File
@@ -0,0 +1,165 @@
package api
import (
"context"
"fmt"
"strings"
"github.com/redis/go-redis/v9"
"github.com/Silo-Server/silo-server/internal/autoscan"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/plugins"
mediarequests "github.com/Silo-Server/silo-server/internal/requests"
"github.com/Silo-Server/silo-server/internal/scantrigger"
)
// autoscanQueuer is the scan-enqueue surface BuildAutoscanService needs; it is
// satisfied by *scanqueue.Service (autoscan.Queuer's concrete production impl).
type autoscanQueuer = autoscan.Queuer
// RequestIntegrationLookup adapts the Requests repository to the autoscan
// connection resolver's RequestIntegrationLookup: it resolves a soft-linked
// Requests integration to its base URL and (write-only) api-key ref.
type RequestIntegrationLookup struct {
Repo *mediarequests.Repository
}
func (l RequestIntegrationLookup) Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error) {
integration, err := l.Repo.GetIntegration(ctx, integrationID)
if err != nil {
return "", "", err
}
// A reused Requests connection must honor the integration's live state. The
// v1 poll gated on `WHERE ri.enabled = true`; here we surface a disabled or
// unconfigured (blank base_url) integration as an error so the engine turns
// it into a logged skip / RecordError rather than polling an unusable target.
if err := checkRequestIntegrationUsable(integrationID, integration.Enabled, integration.BaseURL); err != nil {
return "", "", err
}
return integration.BaseURL, integration.APIKeyRef, nil
}
// checkRequestIntegrationUsable returns a non-nil error when a linked Requests
// integration cannot be polled: it is disabled, or it has no base_url. Extracted
// as a pure function so the gating is unit-testable without a DB-backed repo.
func checkRequestIntegrationUsable(integrationID string, enabled bool, baseURL string) error {
if !enabled {
return fmt.Errorf("linked requests integration %q is disabled", integrationID)
}
if strings.TrimSpace(baseURL) == "" {
return fmt.Errorf("linked requests integration %q has no base_url configured", integrationID)
}
return nil
}
// PluginScanSourceAdapter adapts plugins.Service to autoscan.ScanSourceResolver.
// plugins.Service.ScanSourceClient returns the concrete
// *pluginhost.ScanSourceClient; that concrete type satisfies
// autoscan.PollChangesClient (it has the matching PollChanges method), so the
// adapter declares the exported interface as its return type and returns the
// concrete value (Go has no return-type covariance, so the method signature must
// name the interface exactly to satisfy ScanSourceResolver).
type PluginScanSourceAdapter struct {
Svc *plugins.Service
}
func (a PluginScanSourceAdapter) ScanSourceClient(ctx context.Context, pluginID, capabilityID string) (autoscan.PollChangesClient, error) {
return a.Svc.ScanSourceClientByPluginID(ctx, pluginID, capabilityID)
}
// scanSourceCapabilityType is the plugin capability type autoscan discovery
// enumerates.
const scanSourceCapabilityType = "scan_source.v1"
// PluginScanSourceLister adapts the plugin installation store to
// autoscan.ScanSourceLister: it enumerates every installed scan_source.v1
// capability across ALL installed plugins, regardless of enabled state.
type PluginScanSourceLister struct {
Store *plugins.InstallationStore
}
func (l PluginScanSourceLister) ListScanSources(ctx context.Context) ([]autoscan.DiscoveredSource, error) {
installations, err := l.Store.List(ctx)
if err != nil {
return nil, err
}
var out []autoscan.DiscoveredSource
for _, inst := range installations {
caps, err := l.Store.ListCapabilities(ctx, inst.ID)
if err != nil {
return nil, err
}
for _, c := range caps {
if c == nil || c.Type != scanSourceCapabilityType {
continue
}
out = append(out, autoscan.DiscoveredSource{
PluginID: inst.PluginID,
CapabilityID: c.ID,
DisplayName: scanSourceDisplayName(inst.PluginID, c),
})
}
}
return out, nil
}
// scanSourceDisplayName derives a human-friendly label for a scan_source
// capability: the capability manifest's display_name when present, else the
// plugin id (with the capability id appended when it adds information).
func scanSourceDisplayName(pluginID string, c *plugins.Capability) string {
if c != nil && c.Metadata != nil {
if name, ok := c.Metadata["display_name"].(string); ok && strings.TrimSpace(name) != "" {
return strings.TrimSpace(name)
}
}
switch {
case pluginID != "" && c != nil && c.ID != "":
return pluginID + " / " + c.ID
case pluginID != "":
return pluginID
case c != nil:
return c.ID
default:
return ""
}
}
// AutoscanSecretResolver resolves an encrypted api-key reference to plaintext.
// It is satisfied by the server settings repo (Get(ctx, key) (string, error)).
type AutoscanSecretResolver interface {
Get(ctx context.Context, ref string) (string, error)
}
// BuildAutoscanService wires the v2 autoscan engine from its concrete
// dependencies. Both the HTTP router (manual trigger) and the background poll
// task share this constructor so the adapter wiring lives in exactly one place.
func BuildAutoscanService(
repo *autoscan.Repository,
pluginService *plugins.Service,
installationStore *plugins.InstallationStore,
requestsRepo *mediarequests.Repository,
secrets AutoscanSecretResolver,
folderRepo *catalog.FolderRepository,
queue autoscanQueuer,
redisClient *redis.Client,
) *autoscan.Service {
provider := autoscan.NewPluginProvider(PluginScanSourceAdapter{pluginService})
connRes := autoscan.NewConnectionResolver(RequestIntegrationLookup{requestsRepo}, secrets)
svc := autoscan.NewService(
repo,
provider,
connRes,
scantrigger.NewResolver(folderRepo),
queue,
autoscan.NewRedisSuppressor(redisClient),
PluginScanSourceLister{installationStore},
)
// Wire the connection-test + rewrite-suggester deps: a (long-timeout)
// arr root-folder/status client and a Silo media-folder lister.
svc.SetSuggesterDeps(
autoscan.NewArrRootFolderClient(nil),
autoscan.NewCatalogFolderLister(folderRepo),
)
return svc
}
internal/api/autoscan_wiring_test.go
+24
View File
@@ -0,0 +1,24 @@
package api
import "testing"
// TestCheckRequestIntegrationUsable verifies the Fix 4 gate: a reused Requests
// connection is rejected when the integration is disabled or has no base_url, so
// the autoscan engine skips it instead of polling an unusable target.
func TestCheckRequestIntegrationUsable(t *testing.T) {
t.Run("enabled with base_url is usable", func(t *testing.T) {
if err := checkRequestIntegrationUsable("req-1", true, "http://radarr:7878"); err != nil {
t.Fatalf("expected usable, got %v", err)
}
})
t.Run("disabled is rejected", func(t *testing.T) {
if err := checkRequestIntegrationUsable("req-1", false, "http://radarr:7878"); err == nil {
t.Fatal("expected error for disabled integration")
}
})
t.Run("blank base_url is rejected", func(t *testing.T) {
if err := checkRequestIntegrationUsable("req-1", true, " "); err == nil {
t.Fatal("expected error for blank base_url")
}
})
}
internal/api/handlers/admin.go
+246 -66
View File
@@ -9,6 +9,7 @@ import (
"io"
"log/slog"
"net/http"
"slices"
"sort"
"strconv"
"strings"
@@ -112,20 +113,34 @@ func NewAdminHandler(
// createUserRequest represents the JSON body for POST /admin/users.
type createUserRequest struct {
Username string `json:"username"`
Email string `json:"email"`
Password string `json:"password"`
Role string `json:"role"`
Permissions []string `json:"permissions"`
CreateDefaultProfile bool `json:"create_default_profile"`
DefaultProfileName string `json:"default_profile_name,omitempty"`
LibraryIDs []int `json:"library_ids"`
MaxPlaybackQuality string `json:"max_playback_quality"`
MaxStreams *int `json:"max_streams,omitempty"`
MaxTranscodes *int `json:"max_transcodes,omitempty"`
MaxProfiles *int `json:"max_profiles,omitempty"`
DownloadAllowed *bool `json:"download_allowed,omitempty"`
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
Username string `json:"username"`
Email string `json:"email"`
Password string `json:"password"`
Role string `json:"role"`
Permissions createStringSliceField `json:"permissions"`
CreateDefaultProfile bool `json:"create_default_profile"`
DefaultProfileName string `json:"default_profile_name,omitempty"`
LibraryIDs []int `json:"library_ids"`
MaxPlaybackQuality string `json:"max_playback_quality"`
MaxStreams *int `json:"max_streams,omitempty"`
MaxTranscodes *int `json:"max_transcodes,omitempty"`
MaxProfiles *int `json:"max_profiles,omitempty"`
DownloadAllowed *bool `json:"download_allowed,omitempty"`
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
}
type createStringSliceField struct {
Set bool
Value []string
}
func (f *createStringSliceField) UnmarshalJSON(data []byte) error {
f.Set = true
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
f.Value = []string{}
return nil
}
return json.Unmarshal(data, &f.Value)
}
type updateLibraryIDsField struct {
@@ -390,7 +405,11 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
writeError(w, http.StatusBadRequest, "bad_request", "max_profiles must be at least 1")
return
}
permissions, err := auth.NormalizePermissions(req.Permissions)
permissions := auth.DefaultUserPermissions()
if req.Permissions.Set {
permissions = req.Permissions.Value
}
permissions, err := auth.NormalizePermissions(permissions)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
@@ -463,7 +482,7 @@ func (h *AdminHandler) HandleUpdateUser(w http.ResponseWriter, r *http.Request)
permissions = &normalized
}
err = h.userRepo.Update(r.Context(), id, models.UpdateUserInput{
updateInput := models.UpdateUserInput{
Username: req.Username,
Email: req.Email,
Password: req.Password,
@@ -477,12 +496,31 @@ func (h *AdminHandler) HandleUpdateUser(w http.ResponseWriter, r *http.Request)
MaxProfiles: req.MaxProfiles,
DownloadAllowed: req.DownloadAllowed,
DownloadTranscodeAllowed: req.DownloadTranscodeAllowed,
})
}
var currentUser *models.User
if updateMayRequireSessionRevocation(updateInput) {
currentUser, err = h.userRepo.GetByID(r.Context(), id)
if err != nil {
if auth.IsNotFound(err) {
writeError(w, http.StatusNotFound, "not_found", "User not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch user")
return
}
}
err = h.userRepo.Update(r.Context(), id, updateInput)
if err != nil {
if auth.IsNotFound(err) {
writeError(w, http.StatusNotFound, "not_found", "User not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update user")
return
}
if updateRequiresSessionRevocation(req) {
if updateRequiresSessionRevocation(currentUser, updateInput) {
if err := h.revokeUserSessions(r.Context(), id); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to revoke updated user sessions")
return
@@ -750,13 +788,35 @@ func (h *AdminHandler) HandleListUserProfiles(w http.ResponseWriter, r *http.Req
writeJSON(w, http.StatusOK, resp)
}
func updateRequiresSessionRevocation(req updateUserRequest) bool {
return req.Password != nil ||
req.Role != nil ||
req.Enabled != nil ||
req.LibraryIDs.Set ||
req.Permissions.Set ||
req.MaxPlaybackQuality != nil
func updateMayRequireSessionRevocation(input models.UpdateUserInput) bool {
return input.Password != nil ||
input.Role != nil ||
input.Enabled != nil ||
input.Permissions != nil ||
input.MaxPlaybackQuality != nil
}
func updateRequiresSessionRevocation(current *models.User, input models.UpdateUserInput) bool {
if input.Password != nil {
return true
}
if current == nil {
return updateMayRequireSessionRevocation(input)
}
if input.Role != nil && *input.Role != current.Role {
return true
}
if input.Enabled != nil && *input.Enabled != current.Enabled {
return true
}
if input.Permissions != nil && !slices.Equal(*input.Permissions, current.Permissions) {
return true
}
if input.MaxPlaybackQuality != nil &&
access.NormalizePlaybackQuality(*input.MaxPlaybackQuality) != access.NormalizePlaybackQuality(current.MaxPlaybackQuality) {
return true
}
return false
}
func (h *AdminHandler) revokeUserSessions(ctx context.Context, userID int) error {
@@ -833,6 +893,9 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) {
var resp AdminStats
if h.StatsSource != nil {
if isTruthyQuery(r.URL.Query().Get("refresh")) {
h.StatsSource.Invalidate()
}
stats, err := h.StatsSource.Get(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get stats")
@@ -859,6 +922,15 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, resp)
}
func isTruthyQuery(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
func (h *AdminHandler) invalidateStats(ctx context.Context, channel, eventType, payload string) {
if h.StatsSource != nil {
h.StatsSource.Invalidate()
@@ -1082,6 +1154,7 @@ var sensitiveSettingKeys = map[string]bool{
"mdblist.api_key": true,
"requests.radarr.api_key": true,
"requests.sonarr.api_key": true,
"subtitle_ai.api_key": true,
"watchsync.trakt.client_id": true,
"watchsync.trakt.client_secret": true,
"watchsync.simkl.client_id": true,
@@ -1214,6 +1287,9 @@ type adminDeviceDetailResponse struct {
DeviceID string `json:"device_id"`
DeviceName string `json:"device_name"`
DevicePlatform string `json:"device_platform"`
OverrideCount int `json:"override_count"`
ProfileCount int `json:"profile_count"`
Profiles []adminDeviceProfileSummary `json:"profiles"`
LastUpdated string `json:"last_updated"`
Settings []adminDeviceSettingResponse `json:"settings"`
}
@@ -1456,6 +1532,12 @@ func (h *AdminHandler) HandleUpdateUserDeviceSetting(w http.ResponseWriter, r *h
if existing != nil {
entry.DeviceName = existing.DeviceName
entry.DevicePlatform = existing.DevicePlatform
} else if registered, err := registeredDeviceForProfile(r.Context(), store, profileID, deviceID); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
return
} else if registered != nil {
entry.DeviceName = registered.DeviceName
entry.DevicePlatform = registered.DevicePlatform
}
if err := store.SetDeviceSetting(r.Context(), entry); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update device setting")
@@ -1578,6 +1660,10 @@ func (h *AdminHandler) HandleListDevices(w http.ResponseWriter, r *http.Request)
if err != nil {
return fmt.Errorf("list device settings: %w", err)
}
devices, err := listRegisteredDevices(gctx, store)
if err != nil {
return fmt.Errorf("list devices: %w", err)
}
profileNames, err := listProfileNamesByID(gctx, store)
if err != nil {
slog.Warn("admin list devices profile lookup failed",
@@ -1586,7 +1672,14 @@ func (h *AdminHandler) HandleListDevices(w http.ResponseWriter, r *http.Request)
)
profileNames = map[string]string{}
}
perUser[i] = buildAdminDeviceSummaries(user.ID, user.Username, user.Email, entries, profileNames)
perUser[i] = buildAdminDeviceSummaries(
user.ID,
user.Username,
user.Email,
entries,
devices,
profileNames,
)
return nil
})
}
@@ -1650,6 +1743,11 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
return
}
registeredDevices, err := listRegisteredDevices(r.Context(), store)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
return
}
profileNames, err := listProfileNamesByID(r.Context(), store)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list profiles")
@@ -1662,12 +1760,26 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
deviceEntries = append(deviceEntries, entry)
}
}
if len(deviceEntries) == 0 {
deviceRegistrations := make([]userstore.DeviceEntry, 0)
for _, entry := range registeredDevices {
if entry.DeviceID == deviceID {
deviceRegistrations = append(deviceRegistrations, entry)
}
}
summaries := buildAdminDeviceSummaries(
user.ID,
user.Username,
user.Email,
deviceEntries,
deviceRegistrations,
profileNames,
)
if len(summaries) == 0 {
writeError(w, http.StatusNotFound, "not_found", "Device not found")
return
}
summary := buildAdminDeviceSummaries(user.ID, user.Username, user.Email, deviceEntries, profileNames)[0]
summary := summaries[0]
writeJSON(w, http.StatusOK, adminDeviceDetailResponse{
UserID: user.ID,
Username: user.Username,
@@ -1675,11 +1787,41 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
DeviceID: summary.DeviceID,
DeviceName: summary.DeviceName,
DevicePlatform: summary.DevicePlatform,
OverrideCount: summary.OverrideCount,
ProfileCount: summary.ProfileCount,
Profiles: summary.Profiles,
LastUpdated: summary.LastUpdated,
Settings: buildAdminDeviceSettingsResponse(user.ID, profileNames, deviceEntries).Settings,
})
}
func listRegisteredDevices(ctx context.Context, store userstore.UserStore) ([]userstore.DeviceEntry, error) {
registry, ok := store.(userstore.DeviceRegistry)
if !ok {
return nil, nil
}
return registry.ListDevices(ctx)
}
func registeredDeviceForProfile(
ctx context.Context,
store userstore.UserStore,
profileID string,
deviceID string,
) (*userstore.DeviceEntry, error) {
devices, err := listRegisteredDevices(ctx, store)
if err != nil {
return nil, err
}
for _, device := range devices {
if device.ProfileID == profileID && device.DeviceID == deviceID {
matched := device
return &matched, nil
}
}
return nil, nil
}
func buildAdminDeviceSettingsResponse(userID int, profileNames map[string]string, entries []userstore.DeviceSettingEntry) adminDeviceSettingsListResponse {
resp := adminDeviceSettingsListResponse{
Settings: make([]adminDeviceSettingResponse, 0, len(entries)),
@@ -1705,6 +1847,7 @@ func buildAdminDeviceSummaries(
username string,
email string,
entries []userstore.DeviceSettingEntry,
registeredDevices []userstore.DeviceEntry,
profileNames map[string]string,
) []adminDeviceSummaryResponse {
type profileAccumulator struct {
@@ -1718,10 +1861,10 @@ func buildAdminDeviceSummaries(
}
byDevice := make(map[string]*summary)
for _, entry := range entries {
deviceID := strings.TrimSpace(entry.DeviceID)
ensureDevice := func(deviceID, deviceName, devicePlatform, lastUpdated string) *summary {
if deviceID == "" {
continue
return nil
}
current, ok := byDevice[deviceID]
if !ok {
@@ -1731,49 +1874,86 @@ func buildAdminDeviceSummaries(
Username: username,
Email: email,
DeviceID: deviceID,
DeviceName: entry.DeviceName,
DevicePlatform: entry.DevicePlatform,
LastUpdated: entry.UpdatedAt,
DeviceName: deviceName,
DevicePlatform: devicePlatform,
LastUpdated: lastUpdated,
},
keys: make(map[string]struct{}),
profiles: make(map[string]*profileAccumulator),
}
byDevice[deviceID] = current
}
current.keys[entry.ProfileID+":"+entry.Key] = struct{}{}
if current.device.DeviceName == "" && deviceName != "" {
current.device.DeviceName = deviceName
}
if current.device.DevicePlatform == "" && devicePlatform != "" {
current.device.DevicePlatform = devicePlatform
}
if lastUpdated > current.device.LastUpdated {
current.device.LastUpdated = lastUpdated
if deviceName != "" {
current.device.DeviceName = deviceName
}
if devicePlatform != "" {
current.device.DevicePlatform = devicePlatform
}
}
return current
}
ensureProfile := func(current *summary, profileID, lastUpdated string) *profileAccumulator {
if current == nil || profileID == "" {
return nil
}
profile, exists := current.profiles[profileID]
if !exists {
profile = &profileAccumulator{
summary: adminDeviceProfileSummary{
ProfileID: profileID,
ProfileName: profileNames[profileID],
LastUpdated: lastUpdated,
},
keys: make(map[string]struct{}),
}
current.profiles[profileID] = profile
return profile
}
if lastUpdated > profile.summary.LastUpdated {
profile.summary.LastUpdated = lastUpdated
}
return profile
}
for _, device := range registeredDevices {
deviceID := strings.TrimSpace(device.DeviceID)
profileID := strings.TrimSpace(device.ProfileID)
current := ensureDevice(
deviceID,
device.DeviceName,
device.DevicePlatform,
device.LastSeenAt,
)
ensureProfile(current, profileID, device.LastSeenAt)
}
for _, entry := range entries {
deviceID := strings.TrimSpace(entry.DeviceID)
profileID := strings.TrimSpace(entry.ProfileID)
if profileID != "" {
profile, exists := current.profiles[profileID]
if !exists {
profile = &profileAccumulator{
summary: adminDeviceProfileSummary{
ProfileID: profileID,
ProfileName: profileNames[profileID],
LastUpdated: entry.UpdatedAt,
},
keys: make(map[string]struct{}),
}
current.profiles[profileID] = profile
}
current := ensureDevice(
deviceID,
entry.DeviceName,
entry.DevicePlatform,
entry.UpdatedAt,
)
if current == nil {
continue
}
if profileID != "" && entry.Key != "" {
current.keys[profileID+":"+entry.Key] = struct{}{}
}
profile := ensureProfile(current, profileID, entry.UpdatedAt)
if profile != nil && entry.Key != "" {
profile.keys[entry.Key] = struct{}{}
if entry.UpdatedAt > profile.summary.LastUpdated {
profile.summary.LastUpdated = entry.UpdatedAt
}
}
if current.device.DeviceName == "" && entry.DeviceName != "" {
current.device.DeviceName = entry.DeviceName
}
if current.device.DevicePlatform == "" && entry.DevicePlatform != "" {
current.device.DevicePlatform = entry.DevicePlatform
}
if entry.UpdatedAt > current.device.LastUpdated {
current.device.LastUpdated = entry.UpdatedAt
if entry.DeviceName != "" {
current.device.DeviceName = entry.DeviceName
}
if entry.DevicePlatform != "" {
current.device.DevicePlatform = entry.DevicePlatform
}
}
}
internal/api/handlers/admin_jobs.go
+114 -4
View File
@@ -14,6 +14,7 @@ import (
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/notifications"
)
const adminJobDownloadExpiry = 15 * time.Minute
@@ -23,12 +24,22 @@ type AdminJobArtifactStore interface {
PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error)
}
type AdminJobsHandler struct {
repo *adminjob.Repository
store AdminJobArtifactStore
type adminJobRepository interface {
List(ctx context.Context, opts adminjob.ListJobsOptions) ([]*models.AdminJob, error)
GetByID(ctx context.Context, id string) (*models.AdminJob, error)
Cancel(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error)
CancelQueued(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error)
UpdateProgress(ctx context.Context, id string, current, total int, message string) error
}
func NewAdminJobsHandler(repo *adminjob.Repository, store AdminJobArtifactStore) *AdminJobsHandler {
type AdminJobsHandler struct {
repo adminJobRepository
store AdminJobArtifactStore
CancelRegistry *adminjob.CancelRegistry
RealtimeHub *notifications.Hub
}
func NewAdminJobsHandler(repo adminJobRepository, store AdminJobArtifactStore) *AdminJobsHandler {
return &AdminJobsHandler{repo: repo, store: store}
}
@@ -120,6 +131,105 @@ func (h *AdminJobsHandler) HandleGet(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, adminJobToResponseForClaims(r, job, h.store, claims))
}
func (h *AdminJobsHandler) HandleCancel(w http.ResponseWriter, r *http.Request) {
if h == nil || h.repo == nil {
writeError(w, http.StatusServiceUnavailable, "not_configured", "Admin jobs are not configured")
return
}
id := chi.URLParam(r, "id")
if id == "" {
writeError(w, http.StatusBadRequest, "bad_request", "Job ID is required")
return
}
job, err := h.repo.GetByID(r.Context(), id)
if err != nil {
if errors.Is(err, adminjob.ErrJobNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Job not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
return
}
if job.JobType != adminjob.JobTypeLibraryRefresh {
writeError(w, http.StatusConflict, "not_cancellable", "Only library metadata refresh jobs can be cancelled")
return
}
switch job.Status {
case adminjob.StatusQueued:
cancelled, cancelErr := h.repo.CancelQueued(
r.Context(),
id,
"Library metadata refresh cancelled",
time.Now().UTC().Add(7*24*time.Hour),
)
if cancelErr != nil {
if errors.Is(cancelErr, adminjob.ErrJobNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Job not found")
return
}
if errors.Is(cancelErr, adminjob.ErrJobNotCancellable) {
advanced, loadErr := h.repo.GetByID(r.Context(), id)
if loadErr != nil {
if errors.Is(loadErr, adminjob.ErrJobNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Job not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
return
}
if advanced.Status == adminjob.StatusRunning {
h.requestRunningCancellation(w, r, advanced)
return
}
writeError(w, http.StatusConflict, "not_cancellable", "Job is no longer cancellable")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel job")
return
}
if h.RealtimeHub != nil {
publishEventJob(
r.Context(),
h.RealtimeHub.EventsHub(),
string(notifications.TypeJobCancelled),
cancelled,
)
}
writeJSON(w, http.StatusOK, adminJobToResponse(r, cancelled, h.store))
case adminjob.StatusRunning:
h.requestRunningCancellation(w, r, job)
default:
writeError(w, http.StatusConflict, "not_cancellable", "Job is no longer cancellable")
}
}
func (h *AdminJobsHandler) requestRunningCancellation(w http.ResponseWriter, r *http.Request, job *models.AdminJob) {
if h.CancelRegistry == nil || !h.CancelRegistry.Cancel(job.ID) {
writeError(w, http.StatusConflict, "not_cancellable", "Running job is not cancellable on this node")
return
}
if err := h.repo.UpdateProgress(
r.Context(),
job.ID,
job.ProgressCurrent,
job.ProgressTotal,
"Cancellation requested",
); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to request cancellation")
return
}
updated, err := h.repo.GetByID(r.Context(), job.ID)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
return
}
writeJSON(w, http.StatusAccepted, adminJobToResponse(r, updated, h.store))
}
func adminJobToResponse(r *http.Request, job *models.AdminJob, store AdminJobArtifactStore) adminJobResponse {
resp := adminJobResponse{
ID: job.ID,
internal/api/handlers/admin_jobs_test.go
+201
View File
@@ -2,12 +2,21 @@ package handlers
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/go-chi/chi/v5"
"github.com/Silo-Server/silo-server/internal/adminjob"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/cache"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/notifications"
)
func TestCanReadAdminJob_AdminCanReadAnyJob(t *testing.T) {
@@ -79,3 +88,195 @@ func TestAdminJobToResponseForClaims_NonAdminSanitizesItemRefreshPayloads(t *tes
t.Fatalf("ErrorMessage = %q, want stripped for non-admin", resp.ErrorMessage)
}
}
func TestAdminJobsHandleCancel_RunningRegistryMissDoesNotUpdateProgress(t *testing.T) {
repo := &fakeAdminJobRepository{
job: &models.AdminJob{
ID: "job-1",
JobType: adminjob.JobTypeLibraryRefresh,
Status: adminjob.StatusRunning,
CreatedByUserID: 1,
Message: "Running",
RequestedAt: time.Now().UTC(),
},
}
handler := NewAdminJobsHandler(repo, nil)
rec := httptest.NewRecorder()
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
if rec.Code != http.StatusConflict {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusConflict)
}
if repo.updateProgressCalls != 0 {
t.Fatalf("UpdateProgress called %d times, want 0", repo.updateProgressCalls)
}
}
func TestAdminJobsHandleCancel_QueuedPublishesCancelledEvent(t *testing.T) {
repo := &fakeAdminJobRepository{
job: &models.AdminJob{
ID: "job-1",
JobType: adminjob.JobTypeLibraryRefresh,
Status: adminjob.StatusQueued,
CreatedByUserID: 1,
Message: "Queued",
RequestedAt: time.Now().UTC(),
},
}
hub := notifications.NewHub("test", &cache.NoopEventBus{})
events, unsubscribe := hub.Subscribe()
defer unsubscribe()
handler := NewAdminJobsHandler(repo, nil)
handler.RealtimeHub = hub
rec := httptest.NewRecorder()
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
}
select {
case event := <-events:
if event.Type != notifications.TypeJobCancelled {
t.Fatalf("event.Type = %q, want %q", event.Type, notifications.TypeJobCancelled)
}
if event.Job == nil || event.Job.ID != "job-1" {
t.Fatalf("event.Job = %#v, want job-1", event.Job)
}
case <-time.After(time.Second):
t.Fatal("timed out waiting for job.cancelled event")
}
}
func TestAdminJobsHandleCancel_QueuedSnapshotFallsBackToRunningCancellation(t *testing.T) {
repo := &fakeAdminJobRepository{
job: &models.AdminJob{
ID: "job-1",
JobType: adminjob.JobTypeLibraryRefresh,
Status: adminjob.StatusQueued,
CreatedByUserID: 1,
Message: "Queued",
RequestedAt: time.Now().UTC(),
ProgressCurrent: 3,
ProgressTotal: 10,
},
promoteToRunningOnCancelQueued: true,
}
cancelled := false
registry := adminjob.NewCancelRegistry()
unregister := registry.Register("job-1", func() {
cancelled = true
})
defer unregister()
handler := NewAdminJobsHandler(repo, nil)
handler.CancelRegistry = registry
rec := httptest.NewRecorder()
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
if rec.Code != http.StatusAccepted {
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusAccepted, rec.Body.String())
}
if !cancelled {
t.Fatal("expected running job cancel registry to be invoked")
}
if repo.cancelQueuedCalls != 1 {
t.Fatalf("CancelQueued called %d times, want 1", repo.cancelQueuedCalls)
}
if repo.cancelCalls != 0 {
t.Fatalf("Cancel called %d times, want 0", repo.cancelCalls)
}
if repo.updateProgressCalls != 1 {
t.Fatalf("UpdateProgress called %d times, want 1", repo.updateProgressCalls)
}
if repo.job.Status != adminjob.StatusRunning {
t.Fatalf("job status = %q, want %q", repo.job.Status, adminjob.StatusRunning)
}
if repo.job.Message != "Cancellation requested" {
t.Fatalf("job message = %q, want cancellation request", repo.job.Message)
}
}
func adminJobCancelRequest(id string) *http.Request {
req := httptest.NewRequest(http.MethodPost, "/admin/jobs/"+id+"/cancel", nil)
routeCtx := chi.NewRouteContext()
routeCtx.URLParams.Add("id", id)
return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
}
type fakeAdminJobRepository struct {
job *models.AdminJob
updateProgressCalls int
cancelCalls int
cancelQueuedCalls int
promoteToRunningOnCancelQueued bool
}
func (r *fakeAdminJobRepository) List(context.Context, adminjob.ListJobsOptions) ([]*models.AdminJob, error) {
if r.job == nil {
return nil, nil
}
return []*models.AdminJob{r.job}, nil
}
func (r *fakeAdminJobRepository) GetByID(_ context.Context, id string) (*models.AdminJob, error) {
if r.job == nil || r.job.ID != id {
return nil, adminjob.ErrJobNotFound
}
cp := *r.job
return &cp, nil
}
func (r *fakeAdminJobRepository) Cancel(_ context.Context, id, message string, _ time.Time) (*models.AdminJob, error) {
r.cancelCalls++
if r.job == nil || r.job.ID != id {
return nil, adminjob.ErrJobNotFound
}
if r.job.Status != adminjob.StatusQueued && r.job.Status != adminjob.StatusRunning {
return nil, adminjob.ErrJobNotCancellable
}
cp := *r.job
cp.Status = adminjob.StatusCancelled
cp.Message = message
r.job = &cp
return &cp, nil
}
func (r *fakeAdminJobRepository) CancelQueued(_ context.Context, id, message string, _ time.Time) (*models.AdminJob, error) {
r.cancelQueuedCalls++
if r.job == nil || r.job.ID != id {
return nil, adminjob.ErrJobNotFound
}
if r.promoteToRunningOnCancelQueued {
cp := *r.job
cp.Status = adminjob.StatusRunning
r.job = &cp
return nil, adminjob.ErrJobNotCancellable
}
if r.job.Status != adminjob.StatusQueued {
return nil, adminjob.ErrJobNotCancellable
}
cp := *r.job
cp.Status = adminjob.StatusCancelled
cp.Message = message
r.job = &cp
return &cp, nil
}
func (r *fakeAdminJobRepository) UpdateProgress(_ context.Context, id string, current, total int, message string) error {
r.updateProgressCalls++
if r.job == nil || r.job.ID != id {
return adminjob.ErrJobNotFound
}
if message == "" {
return errors.New("message is required")
}
r.job.ProgressCurrent = current
r.job.ProgressTotal = total
r.job.Message = message
return nil
}
internal/api/handlers/admin_marker_providers.go
+230
View File
@@ -0,0 +1,230 @@
package handlers
import (
"encoding/json"
"log/slog"
"net/http"
"sort"
"github.com/go-chi/chi/v5"
"github.com/Silo-Server/silo-server/internal/cache"
"github.com/Silo-Server/silo-server/internal/markers"
)
// AdminMarkerProvidersHandler serves the per-provider marker config + key
// validation API under the RequireAdmin group.
type AdminMarkerProvidersHandler struct {
Registry *markers.Registry
Config *markers.ProviderConfigStore
EventBus cache.EventBus
logger *slog.Logger
}
// NewAdminMarkerProvidersHandler constructs the handler.
func NewAdminMarkerProvidersHandler(registry *markers.Registry, config *markers.ProviderConfigStore, eventBus cache.EventBus, logger *slog.Logger) *AdminMarkerProvidersHandler {
if logger == nil {
logger = slog.Default()
}
return &AdminMarkerProvidersHandler{Registry: registry, Config: config, EventBus: eventBus, logger: logger}
}
type providerConfigResponse struct {
Provider string `json:"provider"`
DisplayName string `json:"display_name,omitempty"`
SourceType string `json:"source_type,omitempty"`
PluginID string `json:"plugin_id,omitempty"`
PluginInstallationID int `json:"plugin_installation_id,omitempty"`
CapabilityID string `json:"capability_id,omitempty"`
IsSubmitter bool `json:"is_submitter"`
FetchEnabled bool `json:"fetch_enabled"`
FetchPriority int `json:"fetch_priority"`
ContributeEnabled bool `json:"contribute_enabled"`
ContributeAutoLocal bool `json:"contribute_auto_local"`
ContributeMinConfidence float64 `json:"contribute_min_confidence"`
}
type markerUserStatsResponse struct {
Total int `json:"total"`
Accepted int `json:"accepted"`
Pending int `json:"pending"`
Rejected int `json:"rejected"`
AcceptanceRate float64 `json:"acceptance_rate"`
CurrentStreak int `json:"current_streak"`
BestStreak int `json:"best_streak"`
}
func (h *AdminMarkerProvidersHandler) submitterIDs() map[string]bool {
out := map[string]bool{}
if h.Registry == nil {
return out
}
for _, p := range h.Registry.Providers() {
if _, ok := p.(markers.Submitter); ok {
out[p.ID()] = true
}
}
return out
}
func (h *AdminMarkerProvidersHandler) providerDescriptions() map[string]markers.ProviderDescriptor {
out := map[string]markers.ProviderDescriptor{}
if h.Registry == nil {
return out
}
for _, p := range h.Registry.Providers() {
desc := markers.ProviderDescriptor{ID: p.ID()}
if described, ok := p.(markers.DescribedProvider); ok {
desc = described.ProviderDescription()
}
if desc.ID == "" {
desc.ID = p.ID()
}
out[p.ID()] = desc
}
return out
}
// HandleListProviders lists registered providers with their config + capability.
func (h *AdminMarkerProvidersHandler) HandleListProviders(w http.ResponseWriter, r *http.Request) {
if h == nil || h.Config == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
return
}
submitters := h.submitterIDs()
descriptions := h.providerDescriptions()
out := []providerConfigResponse{}
if h.Registry != nil {
for _, provider := range h.Registry.Providers() {
c, ok := h.Config.Get(provider.ID())
if !ok {
continue
}
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
}
} else {
for _, c := range h.Config.List() {
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].FetchPriority != out[j].FetchPriority {
return out[i].FetchPriority < out[j].FetchPriority
}
return out[i].Provider < out[j].Provider
})
writeJSON(w, http.StatusOK, map[string]any{"providers": out})
}
// HandleUpdateProvider updates a provider's config row.
func (h *AdminMarkerProvidersHandler) HandleUpdateProvider(w http.ResponseWriter, r *http.Request) {
if h == nil || h.Config == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
return
}
provider := chi.URLParam(r, "provider")
existing, ok := h.Config.Get(provider)
if !ok {
writeError(w, http.StatusNotFound, "not_found", "Unknown marker provider")
return
}
var body struct {
FetchEnabled *bool `json:"fetch_enabled"`
FetchPriority *int `json:"fetch_priority"`
ContributeEnabled *bool `json:"contribute_enabled"`
ContributeAutoLocal *bool `json:"contribute_auto_local"`
ContributeMinConfidence *float64 `json:"contribute_min_confidence"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
if body.FetchEnabled != nil {
existing.FetchEnabled = *body.FetchEnabled
}
if body.FetchPriority != nil {
existing.FetchPriority = *body.FetchPriority
}
if body.ContributeEnabled != nil {
existing.ContributeEnabled = *body.ContributeEnabled
}
if body.ContributeAutoLocal != nil {
existing.ContributeAutoLocal = *body.ContributeAutoLocal
}
if body.ContributeMinConfidence != nil {
v := *body.ContributeMinConfidence
if v < 0 || v > 1 {
writeError(w, http.StatusBadRequest, "bad_request", "contribute_min_confidence must be between 0 and 1")
return
}
existing.ContributeMinConfidence = v
}
if err := h.Config.Update(r.Context(), existing); err != nil {
h.logger.Error("admin markers: update provider config failed", "provider", provider, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update provider")
return
}
if h.EventBus != nil {
_ = h.EventBus.Publish(r.Context(), cache.ChannelAdmin, cache.Event{
Type: cache.EventMarkerProviderConfigChanged,
Payload: provider,
})
}
writeJSON(w, http.StatusOK, toProviderConfigResponse(existing, h.submitterIDs()[provider], h.providerDescriptions()[provider]))
}
// HandleValidateProvider validates the provider's configured key and returns stats.
func (h *AdminMarkerProvidersHandler) HandleValidateProvider(w http.ResponseWriter, r *http.Request) {
if h == nil || h.Registry == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
return
}
provider := chi.URLParam(r, "provider")
var submitter markers.Submitter
for _, p := range h.Registry.Providers() {
if p.ID() == provider {
if s, ok := p.(markers.Submitter); ok {
submitter = s
}
}
}
if submitter == nil {
writeError(w, http.StatusBadRequest, "bad_request", "Provider does not support contribution")
return
}
stats, err := submitter.FetchUserStats(r.Context())
if err != nil {
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": err.Error()})
return
}
writeJSON(w, http.StatusOK, map[string]any{"valid": true, "stats": toMarkerUserStatsResponse(stats)})
}
func toProviderConfigResponse(c markers.ProviderConfig, isSubmitter bool, desc markers.ProviderDescriptor) providerConfigResponse {
return providerConfigResponse{
Provider: c.Provider,
DisplayName: desc.DisplayName,
SourceType: desc.SourceType,
PluginID: desc.PluginID,
PluginInstallationID: desc.PluginInstallationID,
CapabilityID: desc.CapabilityID,
IsSubmitter: isSubmitter,
FetchEnabled: c.FetchEnabled,
FetchPriority: c.FetchPriority,
ContributeEnabled: c.ContributeEnabled,
ContributeAutoLocal: c.ContributeAutoLocal,
ContributeMinConfidence: c.ContributeMinConfidence,
}
}
func toMarkerUserStatsResponse(s markers.UserStats) markerUserStatsResponse {
return markerUserStatsResponse{
Total: s.Total,
Accepted: s.Accepted,
Pending: s.Pending,
Rejected: s.Rejected,
AcceptanceRate: s.AcceptanceRate,
CurrentStreak: s.CurrentStreak,
BestStreak: s.BestStreak,
}
}
internal/api/handlers/admin_marker_providers_test.go
+73
View File
@@ -0,0 +1,73 @@
package handlers
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/go-chi/chi/v5"
"github.com/Silo-Server/silo-server/internal/markers"
)
type fakeMarkerStatsSubmitter struct{}
func (fakeMarkerStatsSubmitter) ID() string { return "introdb" }
func (fakeMarkerStatsSubmitter) FetchMarkers(context.Context, markers.Request) (markers.Result, error) {
return markers.Result{}, nil
}
func (fakeMarkerStatsSubmitter) SubmitMarker(context.Context, markers.SubmissionRequest) (markers.SubmissionResult, error) {
return markers.SubmissionResult{}, nil
}
func (fakeMarkerStatsSubmitter) FetchUserStats(context.Context) (markers.UserStats, error) {
return markers.UserStats{Total: 10, Accepted: 7, Pending: 2, Rejected: 1, AcceptanceRate: 0.7, CurrentStreak: 3, BestStreak: 5}, nil
}
func TestValidateMarkerProviderUsesSnakeCaseStats(t *testing.T) {
reg := markers.NewRegistry(nil)
if err := reg.Register(fakeMarkerStatsSubmitter{}); err != nil {
t.Fatalf("register provider: %v", err)
}
h := NewAdminMarkerProvidersHandler(reg, nil, nil, nil)
req := httptest.NewRequest(http.MethodPost, "/admin/markers/providers/introdb/validate", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("provider", "introdb")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rec := httptest.NewRecorder()
h.HandleValidateProvider(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
body := rec.Body.String()
if !strings.Contains(body, `"acceptance_rate":0.7`) || strings.Contains(body, "AcceptanceRate") {
t.Fatalf("unexpected stats response shape: %s", body)
}
}
func TestMarkerProviderResponseIncludesPluginMetadata(t *testing.T) {
resp := toProviderConfigResponse(
markers.ProviderConfig{Provider: "plugin:4:markers", FetchEnabled: true, FetchPriority: 25},
true,
markers.ProviderDescriptor{
DisplayName: "Plugin Markers",
SourceType: markers.ProviderSourcePlugin,
PluginID: "silo.plugin.markers",
PluginInstallationID: 4,
CapabilityID: "markers",
},
)
if resp.DisplayName != "Plugin Markers" || resp.SourceType != markers.ProviderSourcePlugin {
t.Fatalf("plugin metadata fields missing: %+v", resp)
}
if resp.PluginID != "silo.plugin.markers" ||
resp.PluginInstallationID != 4 ||
resp.CapabilityID != "markers" ||
!resp.IsSubmitter {
t.Fatalf("plugin identity fields missing: %+v", resp)
}
}
internal/api/handlers/admin_stats.go
+44 -8
View File
@@ -22,7 +22,9 @@ type AdminStats struct {
TotalFiles int `json:"total_files"`
TotalUsers int `json:"total_users"`
TotalMovies int `json:"total_movies"`
TotalMovieFiles int `json:"total_movie_files"`
TotalShows int `json:"total_shows"`
TotalShowFiles int `json:"total_show_files"`
ActiveStreams int `json:"active_streams"`
TotalStorageBytes int64 `json:"total_storage_bytes"`
WatchProviderActivity WatchProviderActivity `json:"watch_provider_activity"`
@@ -127,13 +129,15 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
}
var (
totalUsers int64
totalItems int64
totalFiles int64
totalMovies int64
totalShows int64
activeStreams int64
totalStorage int64
totalUsers int64
totalItems int64
totalFiles int64
totalMovies int64
totalMovieFiles int64
totalShows int64
totalShowFiles int64
activeStreams int64
totalStorage int64
)
row := pool.QueryRow(ctx, `
@@ -151,8 +155,34 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
file_stats AS (
SELECT
COUNT(*)::bigint AS total_files,
COUNT(*) FILTER (WHERE file_kind = 'movie')::bigint AS total_movie_files,
COUNT(*) FILTER (WHERE file_kind = 'series')::bigint AS total_show_files,
COALESCE(SUM(file_size), 0)::bigint AS total_storage_bytes
FROM media_files
FROM (
SELECT
media_files.file_size,
CASE
WHEN lower(trim(COALESCE(NULLIF(media_items.type, ''), ''))) = 'movie'
THEN 'movie'
WHEN lower(trim(COALESCE(NULLIF(media_items.type, ''), ''))) = 'series'
THEN 'series'
WHEN episodes.content_id IS NOT NULL
THEN 'series'
WHEN lower(trim(COALESCE(NULLIF(media_files.base_type, ''), ''))) IN ('movie', 'movies')
THEN 'movie'
WHEN lower(trim(COALESCE(NULLIF(media_files.base_type, ''), ''))) IN ('series', 'tv', 'show', 'shows', 'tvshows')
THEN 'series'
WHEN lower(trim(media_folders.type)) IN ('movie', 'movies')
THEN 'movie'
WHEN lower(trim(media_folders.type)) IN ('series', 'tv', 'show', 'shows', 'tvshows')
THEN 'series'
ELSE ''
END AS file_kind
FROM media_files
JOIN media_folders ON media_folders.id = media_files.media_folder_id
LEFT JOIN media_items ON media_items.content_id = media_files.content_id
LEFT JOIN episodes ON episodes.content_id = media_files.episode_id
) classified_files
),
session_stats AS (
SELECT COUNT(*)::bigint AS active_streams
@@ -163,7 +193,9 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
item_stats.total_items,
file_stats.total_files,
item_stats.total_movies,
file_stats.total_movie_files,
item_stats.total_shows,
file_stats.total_show_files,
session_stats.active_streams,
file_stats.total_storage_bytes
FROM user_stats
@@ -176,7 +208,9 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
&totalItems,
&totalFiles,
&totalMovies,
&totalMovieFiles,
&totalShows,
&totalShowFiles,
&activeStreams,
&totalStorage,
); err != nil {
@@ -194,7 +228,9 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
TotalItems: int(totalItems),
TotalFiles: int(totalFiles),
TotalMovies: int(totalMovies),
TotalMovieFiles: int(totalMovieFiles),
TotalShows: int(totalShows),
TotalShowFiles: int(totalShowFiles),
ActiveStreams: int(activeStreams),
TotalStorageBytes: totalStorage,
WatchProviderActivity: activity,
internal/api/handlers/admin_test.go
+74 -15
View File
@@ -1,73 +1,132 @@
package handlers
import "testing"
import (
"testing"
"github.com/Silo-Server/silo-server/internal/models"
)
func TestUpdateRequiresSessionRevocation(t *testing.T) {
role := "admin"
sameRole := "user"
enabled := true
disabled := false
libraryIDs := []int{1, 2}
sameLibraryIDs := []int{1}
emptyLibraryIDs := []int{}
var allLibraryIDs []int
maxPlaybackQuality := "1080p"
sameMaxPlaybackQuality := "original"
password := "new-password"
username := "renamed"
maxStreams := 4
permissions := []string{"metadata_curation"}
samePermissions := []string{"download"}
current := &models.User{
Role: "user",
Permissions: []string{"download"},
Enabled: false,
LibraryIDs: []int{1},
MaxPlaybackQuality: "original",
}
tests := []struct {
name string
req updateUserRequest
in models.UpdateUserInput
want bool
}{
{
name: "permissions set",
req: updateUserRequest{Permissions: updateStringSliceField{Set: true, Value: []string{"metadata_curation"}}},
in: models.UpdateUserInput{Permissions: &permissions},
want: true,
},
{
name: "permissions unset",
req: updateUserRequest{Permissions: updateStringSliceField{Set: false, Value: []string{"metadata_curation"}}},
name: "permissions unchanged",
in: models.UpdateUserInput{Permissions: &samePermissions},
want: false,
},
{
name: "role",
req: updateUserRequest{Role: &role},
in: models.UpdateUserInput{Role: &role},
want: true,
},
{
name: "role unchanged",
in: models.UpdateUserInput{Role: &sameRole},
want: false,
},
{
name: "enabled",
req: updateUserRequest{Enabled: &enabled},
in: models.UpdateUserInput{Enabled: &enabled},
want: true,
},
{
name: "library ids",
req: updateUserRequest{LibraryIDs: updateLibraryIDsField{Set: true, Value: libraryIDs}},
want: true,
name: "enabled unchanged",
in: models.UpdateUserInput{Enabled: &disabled},
want: false,
},
{
name: "library ids does not revoke session",
in: models.UpdateUserInput{LibraryIDs: &libraryIDs},
want: false,
},
{
name: "library ids unchanged",
in: models.UpdateUserInput{LibraryIDs: &sameLibraryIDs},
want: false,
},
{
name: "library ids nil does not revoke session",
in: models.UpdateUserInput{LibraryIDs: &allLibraryIDs},
want: false,
},
{
name: "max playback quality",
req: updateUserRequest{MaxPlaybackQuality: &maxPlaybackQuality},
in: models.UpdateUserInput{MaxPlaybackQuality: &maxPlaybackQuality},
want: true,
},
{
name: "max playback quality unchanged",
in: models.UpdateUserInput{MaxPlaybackQuality: &sameMaxPlaybackQuality},
want: false,
},
{
name: "password",
req: updateUserRequest{Password: &password},
in: models.UpdateUserInput{Password: &password},
want: true,
},
{
name: "non access fields",
req: updateUserRequest{Username: &username, MaxStreams: &maxStreams},
in: models.UpdateUserInput{Username: &username, MaxStreams: &maxStreams},
want: false,
},
{
name: "empty update",
req: updateUserRequest{},
in: models.UpdateUserInput{},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := updateRequiresSessionRevocation(tt.req); got != tt.want {
if got := updateRequiresSessionRevocation(current, tt.in); got != tt.want {
t.Fatalf("updateRequiresSessionRevocation() = %v, want %v", got, tt.want)
}
})
}
unrestrictedCurrent := *current
unrestrictedCurrent.LibraryIDs = nil
t.Run("library ids empty does not revoke session", func(t *testing.T) {
if got := updateRequiresSessionRevocation(&unrestrictedCurrent, models.UpdateUserInput{LibraryIDs: &emptyLibraryIDs}); got {
t.Fatalf("updateRequiresSessionRevocation() = %v, want false", got)
}
})
t.Run("library ids nil unchanged", func(t *testing.T) {
if got := updateRequiresSessionRevocation(&unrestrictedCurrent, models.UpdateUserInput{LibraryIDs: &allLibraryIDs}); got {
t.Fatalf("updateRequiresSessionRevocation() = %v, want false", got)
}
})
}
internal/api/handlers/auth.go
+14 -3
View File
@@ -17,9 +17,10 @@ import (
// AuthHandler handles authentication-related HTTP endpoints.
type AuthHandler struct {
service *auth.Service
jwt *auth.JWTService
device *auth.DeviceLoginService
service *auth.Service
jwt *auth.JWTService
device *auth.DeviceLoginService
oauthRoutesAvailable bool
}
// NewAuthHandler creates a new AuthHandler backed by the given auth, JWT,
@@ -32,6 +33,13 @@ func NewAuthHandler(service *auth.Service, jwt *auth.JWTService, device *auth.De
}
}
// SetOAuthRoutesAvailable controls whether OAuth login providers are
// advertised by /auth/providers. The router only mounts OAuth routes when the
// server has enough configuration to complete the flow.
func (h *AuthHandler) SetOAuthRoutesAvailable(available bool) {
h.oauthRoutesAvailable = available
}
// --- Request/Response types ---
// loginRequest represents the JSON body of a POST /auth/login request.
@@ -181,6 +189,9 @@ func (h *AuthHandler) HandleProviders(w http.ResponseWriter, r *http.Request) {
providers := h.service.ListProviders()
response := make([]authProviderResponse, 0, len(providers))
for _, provider := range providers {
if provider.Mode == "oauth" && !h.oauthRoutesAvailable {
continue
}
response = append(response, authProviderResponse{
ID: provider.ID,
DisplayName: provider.DisplayName,
internal/api/handlers/auth_providers_test.go
+97
View File
@@ -0,0 +1,97 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/models"
)
type stubLoginProvider struct{}
func (stubLoginProvider) Authenticate(context.Context, auth.Credentials) (*models.User, error) {
return nil, auth.ErrInvalidCredentials
}
func (stubLoginProvider) ValidateSession(context.Context, string) (bool, error) {
return false, nil
}
func newAuthProviderHandlerForTest(oauthRoutesAvailable bool) *AuthHandler {
service := auth.NewService(nil, nil, nil, nil, nil, nil, nil)
provider := stubLoginProvider{}
service.RegisterProvider(auth.LoginProviderInfo{
ID: "local",
DisplayName: "Local",
Mode: "credentials",
Default: true,
}, provider)
service.RegisterProvider(auth.LoginProviderInfo{
ID: "plugin:41:oidc",
DisplayName: "OIDC",
Mode: "oauth",
InstallationID: 41,
}, provider)
handler := NewAuthHandler(service, nil, nil)
handler.SetOAuthRoutesAvailable(oauthRoutesAvailable)
return handler
}
func readProviderResponse(t *testing.T, handler *AuthHandler) []authProviderResponse {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/auth/providers", nil)
handler.HandleProviders(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
}
var providers []authProviderResponse
if err := json.NewDecoder(rec.Body).Decode(&providers); err != nil {
t.Fatalf("decode response: %v", err)
}
return providers
}
func TestAuthProvidersHideOAuthWhenOAuthRoutesUnavailable(t *testing.T) {
providers := readProviderResponse(t, newAuthProviderHandlerForTest(false))
if len(providers) != 1 {
t.Fatalf("provider count = %d, want 1: %#v", len(providers), providers)
}
if providers[0].ID != "local" {
t.Fatalf("provider ID = %q, want local", providers[0].ID)
}
}
func TestAuthProvidersIncludeOAuthWhenOAuthRoutesAvailable(t *testing.T) {
providers := readProviderResponse(t, newAuthProviderHandlerForTest(true))
if len(providers) != 2 {
t.Fatalf("provider count = %d, want 2: %#v", len(providers), providers)
}
var foundOAuth bool
for _, provider := range providers {
if provider.ID == "plugin:41:oidc" {
foundOAuth = true
if provider.Mode != "oauth" {
t.Fatalf("OAuth provider mode = %q, want oauth", provider.Mode)
}
if provider.InstallationID != 41 {
t.Fatalf("OAuth provider installation ID = %d, want 41", provider.InstallationID)
}
}
}
if !foundOAuth {
t.Fatalf("OAuth provider missing from response: %#v", providers)
}
}
internal/api/handlers/autoscan.go
+1044
View File
File diff suppressed because it is too large Load Diff
internal/api/handlers/autoscan_test.go
+1332
View File
File diff suppressed because it is too large Load Diff
internal/api/handlers/catalog.go
+97
View File
@@ -42,6 +42,9 @@ type catalogFiltersResponse struct {
Countries []string `json:"countries"`
OriginalLanguages []string `json:"original_languages"`
ContentRatings []string `json:"content_ratings"`
Authors []string `json:"authors"`
Narrators []string `json:"narrators"`
Series []string `json:"series"`
Resolutions *[]string `json:"resolutions,omitempty"`
AudioLanguages *[]string `json:"audio_languages,omitempty"`
SubtitleLanguages *[]string `json:"subtitle_languages,omitempty"`
@@ -77,6 +80,17 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request
overlaySummaries := h.itemsH.listOverlaySummaries(r.Context(), result.Items, h.itemsH.accessFilter(r))
userStates := h.itemsH.listItemUserStates(r, result.Items)
episodeMetadata := h.itemsH.listEpisodeBrowseMetadata(r.Context(), result.Items)
sortField := catalog.NormalizeQuerySort(req.Query.Sort).Field
store, profileID, _ := h.itemsH.userStoreForRequest(r)
sortMetrics := h.itemsH.listSortMetrics(
r.Context(),
result.Items,
sortField,
h.itemsH.accessFilter(r),
overlaySummaries,
store,
profileID,
)
items := make([]itemListResponse, 0, len(result.Items))
for _, item := range result.Items {
resp := h.itemsH.toItemListResponseWithOverlay(
@@ -90,6 +104,7 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request
resp.SeasonNumber = meta.SeasonNumber
resp.EpisodeNumber = meta.EpisodeNumber
}
resp.SortMetrics = sortMetrics[item.ContentID]
items = append(items, resp)
}
@@ -151,12 +166,94 @@ func (h *CatalogHandler) HandleGetCatalogFilters(w http.ResponseWriter, r *http.
Countries: filters.Countries,
OriginalLanguages: filters.OriginalLanguages,
ContentRatings: filters.ContentRatings,
Authors: filters.Authors,
Narrators: filters.Narrators,
Series: filters.Series,
Resolutions: resolutions,
AudioLanguages: audioLanguages,
SubtitleLanguages: subtitleLanguages,
})
}
// catalogFacetSearchResponse mirrors catalog.CatalogFacetSearchResult on
// the wire. matches[] is always present (empty when no hits); has_more
// is true when the underlying result set held more entries than the
// requested limit.
type catalogFacetSearchResponse struct {
Matches []string `json:"matches"`
HasMore bool `json:"has_more"`
}
// HandleGetCatalogFacetSearch — GET /api/v1/catalog/filters/search
//
// Prefix-typeahead for the high-cardinality filter facets (authors /
// narrators / series, plus genre / studio / network / country /
// original_language / content_rating for consistency). Query
// parameters: same as /api/v1/catalog/filters for scope (source,
// library_id, etc.), plus facet=<name>, q=<prefix>, limit=<N>.
//
// The bulk /api/v1/catalog/filters endpoint stays as the source for
// the initial dropdown render (top 1000 alphabetical); this endpoint
// takes over once the user starts typing.
func (h *CatalogHandler) HandleGetCatalogFacetSearch(w http.ResponseWriter, r *http.Request) {
if h == nil || h.resolver == nil || h.itemsH == nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Catalog is not configured")
return
}
req, err := catalog.ParseCatalogRequest(r.URL.Query())
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
facet := strings.TrimSpace(r.URL.Query().Get("facet"))
if facet == "" {
writeError(w, http.StatusBadRequest, "bad_request", "facet parameter is required")
return
}
prefix := r.URL.Query().Get("q")
limit := 20
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
n, parseErr := strconv.Atoi(raw)
if parseErr != nil || n <= 0 {
writeError(w, http.StatusBadRequest, "bad_request", "limit must be a positive integer")
return
}
if n > 100 {
n = 100
}
limit = n
}
result, err := h.resolver.SearchFacet(
r.Context(),
req,
h.itemsH.accessFilter(r),
facet,
prefix,
limit,
)
if err != nil {
if errors.Is(err, catalog.ErrInvalidCatalogRequest) {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to search catalog facet")
return
}
matches := result.Matches
if matches == nil {
matches = []string{}
}
writeJSON(w, http.StatusOK, catalogFacetSearchResponse{
Matches: matches,
HasMore: result.HasMore,
})
}
func parseIncludeTechnical(raw string) bool {
if strings.TrimSpace(raw) == "" {
return true
internal/api/handlers/catalog_resources.go
+1 -1
View File
@@ -497,7 +497,7 @@ func (h *CatalogResourceHandler) enrichItemDetail(r *http.Request, detail *catal
detail.SeasonUserData = h.items.getAggregateUserData(r, episodes)
}
}
case "movie", "episode":
case "movie", "episode", "audiobook":
detail.SeasonUserData = h.items.getLeafUserData(r, detail.ContentID)
applyEffectiveEditionPreference(detail.SeasonUserData, &detail.EffectiveVersionEditionKey)
}
internal/api/handlers/events_publish.go
+2 -1
View File
@@ -23,9 +23,10 @@ func publishEventMetadataUpdate(ctx context.Context, hub *evt.Hub, libraryID int
if hub == nil {
return
}
if err := hub.PublishJSON(ctx, evt.ChannelCatalog, "metadata.updated", map[string]any{
if err := hub.PublishJSON(ctx, evt.ChannelCatalog, "catalog.item.changed", map[string]any{
"library_id": libraryID,
"content_id": contentID,
"change": "metadata_updated",
}, evt.PublishOptions{}); err != nil {
slog.Warn("events: failed to publish metadata update", "library_id", libraryID, "content_id", contentID, "error", err)
}
internal/api/handlers/events_ws.go
+10 -6
View File
@@ -391,13 +391,17 @@ func (h *EventsHandler) writeSnapshotFrame(
) error {
data, err := h.snapshotForChannel(r, claims, channel)
if err != nil {
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
_ = writeWebSocketControl(
conn,
websocket.CloseMessage,
websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "snapshot failed"),
slog.Error(
"events: failed to build initial snapshot",
"channel",
channel,
"user_id",
claims.UserID,
"error",
err,
)
return err
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
return nil
}
return writeWebSocketJSON(conn, evt.EventsSnapshotMessage{
Type: "snapshot",
internal/api/handlers/favorites.go
+13 -5
View File
@@ -216,7 +216,9 @@ func (h *PersonalDataHandler) HandleAddFavorite(w http.ResponseWriter, r *http.R
h.dispatchLocalFavoriteEvent(r.Context(), watchsync.LocalFavoriteEventAdded, userID, profileID, itemID)
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", "favorite.updated")
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", userStateEventState{
IsFavorite: boolPtr(true),
})
w.WriteHeader(http.StatusNoContent)
}
@@ -244,7 +246,9 @@ func (h *PersonalDataHandler) HandleRemoveFavorite(w http.ResponseWriter, r *htt
h.dispatchLocalFavoriteEvent(r.Context(), watchsync.LocalFavoriteEventRemoved, userID, profileID, itemID)
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", "favorite.updated")
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", userStateEventState{
IsFavorite: boolPtr(false),
})
w.WriteHeader(http.StatusNoContent)
}
@@ -366,7 +370,9 @@ func (h *PersonalDataHandler) HandleAddToWatchlist(w http.ResponseWriter, r *htt
}
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", "watchlist.updated")
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", userStateEventState{
InWatchlist: boolPtr(true),
})
w.WriteHeader(http.StatusNoContent)
}
@@ -393,7 +399,9 @@ func (h *PersonalDataHandler) HandleRemoveFromWatchlist(w http.ResponseWriter, r
}
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", "watchlist.updated")
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", userStateEventState{
InWatchlist: boolPtr(false),
})
w.WriteHeader(http.StatusNoContent)
}
@@ -492,7 +500,7 @@ func (h *PersonalDataHandler) HandleRemoveHistory(w http.ResponseWriter, r *http
mediaItemID,
"",
"history",
"history.updated",
userStateEventState{},
)
}
w.WriteHeader(http.StatusNoContent)
internal/api/handlers/home_dismissals.go
+22
View File
@@ -8,11 +8,13 @@ import (
"github.com/go-chi/chi/v5"
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
evt "github.com/Silo-Server/silo-server/internal/events"
"github.com/Silo-Server/silo-server/internal/userstore"
)
type HomeDismissalHandler struct {
storeProvider userstore.UserStoreProvider
EventsHub *evt.Hub
}
type upsertHomeDismissalRequest struct {
@@ -76,6 +78,16 @@ func (h *HomeDismissalHandler) HandleUpsertDismissal(w http.ResponseWriter, r *h
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save dismissal")
return
}
publishUserStateEvent(
r.Context(),
h.EventsHub,
userID,
profileID,
itemID,
req.SeriesID,
"home_dismissal",
userStateEventState{},
)
w.WriteHeader(http.StatusNoContent)
}
@@ -104,6 +116,16 @@ func (h *HomeDismissalHandler) HandleDeleteDismissal(w http.ResponseWriter, r *h
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete dismissal")
return
}
publishUserStateEvent(
r.Context(),
h.EventsHub,
userID,
profileID,
itemID,
"",
"home_dismissal",
userStateEventState{},
)
w.WriteHeader(http.StatusNoContent)
}
internal/api/handlers/items.go
+252 -15
View File
@@ -34,6 +34,10 @@ type EpisodeFileProvider interface {
ListByContentIDs(ctx context.Context, contentIDs []string) (map[string][]*models.MediaFile, error)
}
type batchEpisodeFileProvider interface {
ListByEpisodeIDs(ctx context.Context, episodeIDs []string) (map[string][]*models.MediaFile, error)
}
type MetadataRefreshRequester interface {
RequestStaleMetadataRefresh(ctx context.Context, targetType, contentID string) error
}
@@ -146,7 +150,7 @@ func (h *ItemsHandler) maybeRequestStaleEpisodeMetadataRefresh(ctx context.Conte
if h == nil || h.metadataRefreshRequester == nil || episode == nil {
return
}
if metadata.EpisodeHasIncompleteMetadata(episode, time.Now()) {
if metadata.EpisodeHasActionableMetadataDebt(episode, time.Now()) {
h.requestStaleMetadataRefresh(ctx, "episode", episode.ContentID)
}
}
@@ -157,7 +161,7 @@ func (h *ItemsHandler) maybeRequestStaleSeasonMetadataRefresh(ctx context.Contex
}
now := time.Now()
for _, episode := range episodes {
if metadata.EpisodeHasIncompleteMetadata(episode, now) {
if metadata.EpisodeHasActionableMetadataDebt(episode, now) {
h.requestStaleMetadataRefresh(ctx, "season", seasonID)
return
}
@@ -209,9 +213,20 @@ type itemListResponse struct {
LastAirDate *string `json:"last_air_date,omitempty"`
AddedAt *time.Time `json:"added_at,omitempty"`
OverlaySummary *models.OverlaySummary `json:"overlay_summary,omitempty"`
SortMetrics *sortMetricsResponse `json:"sort_metrics,omitempty"`
UserState *itemUserStateResponse `json:"user_state,omitempty"`
}
type sortMetricsResponse struct {
ReleaseDate *string `json:"release_date,omitempty"`
RuntimeMinutes *int `json:"runtime_minutes,omitempty"`
Resolution string `json:"resolution,omitempty"`
BitrateKbps *int `json:"bitrate_kbps,omitempty"`
ProgressRatio *float64 `json:"progress_ratio,omitempty"`
ViewedAt string `json:"viewed_at,omitempty"`
PlayCount *int `json:"play_count,omitempty"`
}
// browseResponse is the paginated response for the /items endpoint.
type browseResponse struct {
Total int `json:"total"`
@@ -461,7 +476,9 @@ func (h *ItemsHandler) handleSetWatchedState(w http.ResponseWriter, r *http.Requ
}
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, id, "", "watched", "watched.updated")
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, id, "", "watched", userStateEventState{
Played: boolPtr(played),
})
writeJSON(w, http.StatusOK, watchedStateResponse{
ContentID: id,
@@ -817,6 +834,147 @@ func (h *ItemsHandler) listOverlaySummaries(ctx context.Context, items []*models
return summaries
}
groupedFiles := h.listBrowseItemFiles(ctx, items, filter)
for contentID, files := range groupedFiles {
if summary := overlays.BuildSummary(files); summary != nil {
summaries[contentID] = summary
}
}
return summaries
}
func (h *ItemsHandler) listSortMetrics(
ctx context.Context,
items []*models.MediaItem,
sortField string,
filter catalog.AccessFilter,
overlaySummaries map[string]*models.OverlaySummary,
store userstore.UserStore,
profileID string,
) map[string]*sortMetricsResponse {
metrics := make(map[string]*sortMetricsResponse, len(items))
switch sortField {
case "release_date":
for _, item := range items {
if item == nil {
continue
}
releaseDate := firstNonBlankPtr(item.ReleaseDate, item.FirstAirDate)
if releaseDate != nil {
metrics[item.ContentID] = &sortMetricsResponse{ReleaseDate: releaseDate}
}
}
case "runtime":
for _, item := range items {
if item == nil || item.Runtime <= 0 {
continue
}
runtimeMinutes := item.Runtime
metrics[item.ContentID] = &sortMetricsResponse{RuntimeMinutes: &runtimeMinutes}
}
case "resolution":
for _, item := range items {
if item == nil {
continue
}
if summary := overlaySummaries[item.ContentID]; summary != nil && strings.TrimSpace(summary.Resolution) != "" {
metrics[item.ContentID] = &sortMetricsResponse{Resolution: summary.Resolution}
}
}
case "bitrate":
groupedFiles := h.listBrowseItemFiles(ctx, items, filter)
for contentID, files := range groupedFiles {
if bitrate := maxFileBitrate(files); bitrate > 0 {
value := bitrate
metrics[contentID] = &sortMetricsResponse{BitrateKbps: &value}
}
}
case "progress", "date_viewed", "plays":
h.listUserSortMetrics(ctx, items, sortField, store, profileID, metrics)
}
return metrics
}
func (h *ItemsHandler) listUserSortMetrics(
ctx context.Context,
items []*models.MediaItem,
sortField string,
store userstore.UserStore,
profileID string,
metrics map[string]*sortMetricsResponse,
) {
if store == nil || profileID == "" || len(items) == 0 {
return
}
contentIDs := uniqueItemContentIDs(items)
if len(contentIDs) == 0 {
return
}
progressMap, err := store.ListProgressByMediaItems(ctx, profileID, contentIDs)
if err != nil {
return
}
switch sortField {
case "progress":
for _, item := range items {
if item == nil || item.ContentID == "" {
continue
}
progress, ok := progressMap[item.ContentID]
if !ok || progress.Completed || progress.PositionSeconds <= 0 || progress.DurationSeconds <= 0 {
continue
}
ratio := progress.PositionSeconds / progress.DurationSeconds
metrics[item.ContentID] = &sortMetricsResponse{ProgressRatio: &ratio}
}
case "date_viewed", "plays":
history, err := listCompletedHistoryForItems(ctx, store, profileID, contentIDs)
if err != nil {
return
}
historyCounts := make(map[string]int, len(contentIDs))
historyViewedAt := make(map[string]string, len(contentIDs))
for _, entry := range history {
if entry.MediaItemID == "" {
continue
}
historyCounts[entry.MediaItemID]++
if entry.WatchedAt > historyViewedAt[entry.MediaItemID] {
historyViewedAt[entry.MediaItemID] = entry.WatchedAt
}
}
for _, item := range items {
if item == nil || item.ContentID == "" {
continue
}
resp := &sortMetricsResponse{}
if sortField == "date_viewed" {
viewedAt := historyViewedAt[item.ContentID]
if progress, ok := progressMap[item.ContentID]; ok && progress.Completed && progress.UpdatedAt > viewedAt {
viewedAt = progress.UpdatedAt
}
if viewedAt == "" {
continue
}
resp.ViewedAt = viewedAt
} else {
playCount := historyCounts[item.ContentID]
if progress, ok := progressMap[item.ContentID]; ok && progress.Completed && playCount < 1 {
playCount = 1
}
if playCount <= 0 {
continue
}
resp.PlayCount = &playCount
}
metrics[item.ContentID] = resp
}
}
}
func uniqueItemContentIDs(items []*models.MediaItem) []string {
contentIDs := make([]string, 0, len(items))
seen := make(map[string]struct{}, len(items))
for _, item := range items {
@@ -829,21 +987,100 @@ func (h *ItemsHandler) listOverlaySummaries(ctx context.Context, items []*models
seen[item.ContentID] = struct{}{}
contentIDs = append(contentIDs, item.ContentID)
}
if len(contentIDs) == 0 {
return summaries
}
return contentIDs
}
groupedFiles, err := h.fileRepo.ListByContentIDs(ctx, contentIDs)
if err != nil {
return summaries
}
for contentID, files := range groupedFiles {
files = catalog.FilterMediaFilesByAccess(files, filter)
if summary := overlays.BuildSummary(files); summary != nil {
summaries[contentID] = summary
func listCompletedHistoryForItems(
ctx context.Context,
store userstore.UserStore,
profileID string,
contentIDs []string,
) ([]userstore.WatchHistoryEntry, error) {
const pageSize = 500
var all []userstore.WatchHistoryEntry
for offset := 0; ; offset += pageSize {
page, err := store.ListCompletedHistory(ctx, userstore.CompletedHistoryQuery{
ProfileID: profileID,
MediaItemIDs: contentIDs,
Limit: pageSize,
Offset: offset,
})
if err != nil {
return nil, err
}
all = append(all, page...)
if len(page) < pageSize {
return all, nil
}
}
return summaries
}
func (h *ItemsHandler) listBrowseItemFiles(ctx context.Context, items []*models.MediaItem, filter catalog.AccessFilter) map[string][]*models.MediaFile {
grouped := make(map[string][]*models.MediaFile, len(items))
if h.fileRepo == nil || len(items) == 0 {
return grouped
}
contentIDs := make([]string, 0, len(items))
episodeIDs := make([]string, 0)
seen := make(map[string]struct{}, len(items))
for _, item := range items {
if item == nil || item.ContentID == "" {
continue
}
if _, ok := seen[item.ContentID]; ok {
continue
}
seen[item.ContentID] = struct{}{}
if item.Type == "episode" {
episodeIDs = append(episodeIDs, item.ContentID)
} else {
contentIDs = append(contentIDs, item.ContentID)
}
}
if len(contentIDs) > 0 {
contentFiles, err := h.fileRepo.ListByContentIDs(ctx, contentIDs)
if err == nil {
for contentID, files := range contentFiles {
grouped[contentID] = catalog.FilterMediaFilesByAccess(files, filter)
}
}
}
if len(episodeIDs) > 0 {
if batchProvider, ok := h.fileRepo.(batchEpisodeFileProvider); ok {
episodeFiles, err := batchProvider.ListByEpisodeIDs(ctx, episodeIDs)
if err == nil {
for episodeID, files := range episodeFiles {
grouped[episodeID] = catalog.FilterMediaFilesByAccess(files, filter)
}
}
}
}
return grouped
}
func firstNonBlankPtr(values ...*string) *string {
for _, value := range values {
if value == nil || strings.TrimSpace(*value) == "" {
continue
}
copyValue := *value
return &copyValue
}
return nil
}
func maxFileBitrate(files []*models.MediaFile) int {
maxBitrate := 0
for _, file := range files {
if file != nil && file.Bitrate > maxBitrate {
maxBitrate = file.Bitrate
}
}
return maxBitrate
}
// toSeasonResponse converts a Season model to an API response.
internal/api/handlers/items_sort_metrics_test.go
+59
View File
@@ -0,0 +1,59 @@
package handlers
import (
"context"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/userstore"
)
func TestListSortMetricsIncludesUserStateSorts(t *testing.T) {
ctx := context.Background()
store := newProfileTestStore(t)
items := []*models.MediaItem{
{ContentID: "movie-progress", Type: "movie", Title: "Progress"},
{ContentID: "movie-viewed", Type: "movie", Title: "Viewed"},
{ContentID: "movie-plays", Type: "movie", Title: "Plays"},
}
handler := &ItemsHandler{}
if err := store.SetProgress(ctx, "profile-1", "movie-progress", 900, 3600, userstore.ProgressThresholds{}); err != nil {
t.Fatalf("SetProgress(progress): %v", err)
}
completedAt := time.Date(2026, 5, 29, 18, 30, 0, 0, time.UTC)
if err := store.SetProgressAt(ctx, "profile-1", "movie-viewed", 3600, 3600, true, completedAt); err != nil {
t.Fatalf("SetProgressAt(viewed): %v", err)
}
for _, watchedAt := range []string{"2026-05-27T10:00:00Z", "2026-05-28T10:00:00Z"} {
if err := store.AddHistory(ctx, userstore.WatchHistoryEntry{
ProfileID: "profile-1",
MediaItemID: "movie-plays",
WatchedAt: watchedAt,
DurationSeconds: 3600,
Completed: true,
Source: userstore.WatchHistorySourcePlayback,
}); err != nil {
t.Fatalf("AddHistory(%s): %v", watchedAt, err)
}
}
progressMetrics := handler.listSortMetrics(ctx, items, "progress", catalog.AccessFilter{}, nil, store, "profile-1")
progress := progressMetrics["movie-progress"]
if progress == nil || progress.ProgressRatio == nil || *progress.ProgressRatio != 0.25 {
t.Fatalf("progress metrics = %#v", progress)
}
viewedMetrics := handler.listSortMetrics(ctx, items, "date_viewed", catalog.AccessFilter{}, nil, store, "profile-1")
if got := viewedMetrics["movie-viewed"]; got == nil || got.ViewedAt != "2026-05-29T18:30:00Z" {
t.Fatalf("viewed metrics = %#v", got)
}
playsMetrics := handler.listSortMetrics(ctx, items, "plays", catalog.AccessFilter{}, nil, store, "profile-1")
plays := playsMetrics["movie-plays"]
if plays == nil || plays.PlayCount == nil || *plays.PlayCount != 2 {
t.Fatalf("plays metrics = %#v", plays)
}
}
internal/api/handlers/libraries.go
+396 -1
View File
@@ -51,6 +51,10 @@ type LibraryHandler struct {
PluginInstallations pluginInstallationLister
SkippedRootRepo *metadata.SkippedRootRepository
StaleIDRepo *metadata.StaleMediaIDRepository
MovieMatchQueueRepo libraryMovieMatchQueue
SeriesMatchQueueRepo librarySeriesMatchQueue
RawMatchBacklogRepo libraryRawMatchBacklog
TVSeriesRootQueue bool
ScannedGroupRepo *scanner.ScannedGroupRepository
GroupOverrideRepo *scanner.MediaGroupOverrideRepository
ObservedLocationRepo *scanner.ObservedLocationRepository
@@ -94,6 +98,27 @@ type libraryScanQueuer interface {
CancelByLibrary(ctx context.Context, libraryID int) (int, error)
}
type libraryMovieMatchQueue interface {
SyncForFolder(ctx context.Context, folderID int) error
DeleteByFolder(ctx context.Context, folderID int) (int, error)
CountByFolder(ctx context.Context, folderID int) (int, error)
ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.MovieMatchQueueEntry, int, error)
}
type librarySeriesMatchQueue interface {
SyncForFolder(ctx context.Context, folderID int) error
DeleteByFolder(ctx context.Context, folderID int) (int, error)
CountByFolder(ctx context.Context, folderID int) (int, error)
ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.SeriesRootMatchQueueEntry, int, error)
}
type libraryRawMatchBacklog interface {
CountUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error)
ListUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode, limit int, offset int) ([]*models.MediaFile, int, error)
SuppressUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error)
RetryUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error)
}
// NewLibraryHandler creates a new LibraryHandler backed by the given folder
// repository and ingest executor. The ingester may be nil if scan endpoints are not needed.
func NewLibraryHandler(
@@ -1061,6 +1086,9 @@ func (h *LibraryHandler) recordAcceptedScan(scanID string, target *scantrigger.T
Trigger: target.Trigger,
Status: "accepted",
})
if run, ok := h.ScanRegistry.Get(scanID); ok {
h.publishScanEvent(context.Background(), "scan.accepted", run)
}
}
func (h *LibraryHandler) markScanRunning(scanID string) {
@@ -1279,6 +1307,371 @@ type refreshLibraryMetadataRequest struct {
Mode string `json:"mode"`
}
type libraryMetadataMatchQueueStatusResponse struct {
LibraryID int `json:"library_id"`
MovieCount int `json:"movie_count"`
SeriesCount int `json:"series_count"`
RawFileCount int `json:"raw_file_count"`
TotalCount int `json:"total_count"`
}
type libraryMetadataMatchQueueActionResponse struct {
Status string `json:"status"`
LibraryID int `json:"library_id"`
MovieCancelled int `json:"movie_cancelled,omitempty"`
SeriesCancelled int `json:"series_cancelled,omitempty"`
RawFileCancelled int `json:"raw_file_cancelled,omitempty"`
RawFileRetried int `json:"raw_file_retried,omitempty"`
TotalCancelled int `json:"total_cancelled,omitempty"`
Queue libraryMetadataMatchQueueStatusResponse `json:"queue"`
}
type libraryMetadataMatchQueueDetailResponse struct {
libraryMetadataMatchQueueStatusResponse
Movies []libraryMovieMatchQueueEntryResponse `json:"movies"`
Series []librarySeriesMatchQueueEntryResponse `json:"series"`
RawFiles []libraryRawMatchBacklogEntryResponse `json:"raw_files"`
}
type libraryMovieMatchQueueEntryResponse struct {
MediaFileID int `json:"media_file_id"`
MediaFolderID int `json:"media_folder_id"`
FilePath string `json:"file_path"`
FirstQueuedAt time.Time `json:"first_queued_at"`
AvailableAt time.Time `json:"available_at"`
LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"`
AttemptCount int `json:"attempt_count"`
LastError string `json:"last_error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
type librarySeriesMatchQueueEntryResponse struct {
MediaFolderID int `json:"media_folder_id"`
ObservedRootPath string `json:"observed_root_path"`
FirstQueuedAt time.Time `json:"first_queued_at"`
AvailableAt time.Time `json:"available_at"`
LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"`
AttemptCount int `json:"attempt_count"`
LastError string `json:"last_error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
type libraryRawMatchBacklogEntryResponse struct {
MediaFileID int `json:"media_file_id"`
MediaFolderID int `json:"media_folder_id"`
FilePath string `json:"file_path"`
BaseTitle string `json:"base_title,omitempty"`
BaseYear int `json:"base_year,omitempty"`
BaseType string `json:"base_type,omitempty"`
LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (h *LibraryHandler) HandleListMetadataMatchQueues(w http.ResponseWriter, r *http.Request) {
if h.folderRepo == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Library repository is not configured")
return
}
folders, err := h.folderRepo.List(r.Context())
if err != nil {
slog.Error("metadata queue: failed to list libraries", "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queues")
return
}
resp := make([]libraryMetadataMatchQueueStatusResponse, 0, len(folders))
for _, folder := range folders {
if folder == nil {
continue
}
status, err := h.metadataMatchQueueStatus(r.Context(), folder.ID)
if err != nil {
slog.Error("metadata queue: failed to load queue status", "library_id", folder.ID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
return
}
resp = append(resp, status)
}
writeJSON(w, http.StatusOK, resp)
}
func (h *LibraryHandler) HandleGetMetadataMatchQueue(w http.ResponseWriter, r *http.Request) {
if !h.metadataMatchBacklogConfigured() {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Metadata matcher backlog is not configured")
return
}
id, err := parseIDParam(r)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid library ID")
return
}
if _, err := h.folderRepo.GetByID(r.Context(), id); err != nil {
if errors.Is(err, catalog.ErrFolderNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Library not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch library")
return
}
limit := 10
if value := strings.TrimSpace(r.URL.Query().Get("limit")); value != "" {
if parsed, parseErr := strconv.Atoi(value); parseErr == nil && parsed > 0 && parsed <= 50 {
limit = parsed
}
}
status, err := h.metadataMatchQueueStatus(r.Context(), id)
if err != nil {
slog.Error("metadata queue: failed to load queue status", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
return
}
resp := libraryMetadataMatchQueueDetailResponse{
libraryMetadataMatchQueueStatusResponse: status,
Movies: []libraryMovieMatchQueueEntryResponse{},
Series: []librarySeriesMatchQueueEntryResponse{},
RawFiles: []libraryRawMatchBacklogEntryResponse{},
}
if h.MovieMatchQueueRepo != nil {
movies, _, err := h.MovieMatchQueueRepo.ListByFolder(r.Context(), id, limit, 0)
if err != nil {
slog.Error("metadata queue: failed to list movie queue", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queue")
return
}
for _, entry := range movies {
resp.Movies = append(resp.Movies, libraryMovieMatchQueueEntryResponse{
MediaFileID: entry.MediaFileID,
MediaFolderID: entry.MediaFolderID,
FilePath: entry.FilePath,
FirstQueuedAt: entry.FirstQueuedAt,
AvailableAt: entry.AvailableAt,
LastAttemptedAt: entry.LastAttemptedAt,
AttemptCount: entry.AttemptCount,
LastError: entry.LastError,
UpdatedAt: entry.UpdatedAt,
})
}
}
if h.SeriesMatchQueueRepo != nil {
series, _, err := h.SeriesMatchQueueRepo.ListByFolder(r.Context(), id, limit, 0)
if err != nil {
slog.Error("metadata queue: failed to list series queue", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queue")
return
}
for _, entry := range series {
resp.Series = append(resp.Series, librarySeriesMatchQueueEntryResponse{
MediaFolderID: entry.MediaFolderID,
ObservedRootPath: entry.ObservedRootPath,
FirstQueuedAt: entry.FirstQueuedAt,
AvailableAt: entry.AvailableAt,
LastAttemptedAt: entry.LastAttemptedAt,
AttemptCount: entry.AttemptCount,
LastError: entry.LastError,
UpdatedAt: entry.UpdatedAt,
})
}
}
if h.RawMatchBacklogRepo != nil {
rawFiles, _, err := h.RawMatchBacklogRepo.ListUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode(), limit, 0)
if err != nil {
slog.Error("metadata queue: failed to list raw backlog", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher backlog")
return
}
for _, file := range rawFiles {
if file == nil {
continue
}
resp.RawFiles = append(resp.RawFiles, libraryRawMatchBacklogEntryResponse{
MediaFileID: file.ID,
MediaFolderID: file.MediaFolderID,
FilePath: file.FilePath,
BaseTitle: file.BaseTitle,
BaseYear: file.BaseYear,
BaseType: file.BaseType,
LastAttemptedAt: file.MatchAttemptedAt,
CreatedAt: file.CreatedAt,
UpdatedAt: file.UpdatedAt,
})
}
}
writeJSON(w, http.StatusOK, resp)
}
func (h *LibraryHandler) HandleRetryMetadataMatchQueue(w http.ResponseWriter, r *http.Request) {
if !h.metadataMatchBacklogConfigured() {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Metadata matcher backlog is not configured")
return
}
id, err := parseIDParam(r)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid library ID")
return
}
if _, err := h.folderRepo.GetByID(r.Context(), id); err != nil {
if errors.Is(err, catalog.ErrFolderNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Library not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch library")
return
}
if h.SeriesMatchQueueRepo != nil {
if err := h.SeriesMatchQueueRepo.SyncForFolder(r.Context(), id); err != nil {
slog.Error("metadata queue: failed to retry series queue", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher")
return
}
}
if h.MovieMatchQueueRepo != nil {
if err := h.MovieMatchQueueRepo.SyncForFolder(r.Context(), id); err != nil {
slog.Error("metadata queue: failed to retry movie queue", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher")
return
}
}
rawFileRetried := 0
if h.RawMatchBacklogRepo != nil {
rawFileRetried, err = h.RawMatchBacklogRepo.RetryUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode())
if err != nil {
slog.Error("metadata queue: failed to retry raw backlog", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher")
return
}
}
status, err := h.metadataMatchQueueStatus(r.Context(), id)
if err != nil {
slog.Error("metadata queue: failed to load retried queue status", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
return
}
writeJSON(w, http.StatusOK, libraryMetadataMatchQueueActionResponse{
Status: "queued",
LibraryID: id,
RawFileRetried: rawFileRetried,
Queue: status,
})
}
func (h *LibraryHandler) HandleCancelMetadataMatchQueue(w http.ResponseWriter, r *http.Request) {
if !h.metadataMatchBacklogConfigured() {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Metadata matcher backlog is not configured")
return
}
id, err := parseIDParam(r)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid library ID")
return
}
if _, err := h.folderRepo.GetByID(r.Context(), id); err != nil {
if errors.Is(err, catalog.ErrFolderNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Library not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch library")
return
}
seriesCancelled := 0
if h.SeriesMatchQueueRepo != nil {
seriesCancelled, err = h.SeriesMatchQueueRepo.DeleteByFolder(r.Context(), id)
if err != nil {
slog.Error("metadata queue: failed to cancel series queue", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel metadata matcher")
return
}
}
movieCancelled := 0
if h.MovieMatchQueueRepo != nil {
movieCancelled, err = h.MovieMatchQueueRepo.DeleteByFolder(r.Context(), id)
if err != nil {
slog.Error("metadata queue: failed to cancel movie queue", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel metadata matcher")
return
}
}
rawFileCancelled := 0
if h.RawMatchBacklogRepo != nil {
rawFileCancelled, err = h.RawMatchBacklogRepo.SuppressUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode())
if err != nil {
slog.Error("metadata queue: failed to suppress raw backlog", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel metadata matcher")
return
}
}
status, err := h.metadataMatchQueueStatus(r.Context(), id)
if err != nil {
slog.Error("metadata queue: failed to load cancelled queue status", "library_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
return
}
writeJSON(w, http.StatusOK, libraryMetadataMatchQueueActionResponse{
Status: "cancelled",
LibraryID: id,
MovieCancelled: movieCancelled,
SeriesCancelled: seriesCancelled,
RawFileCancelled: rawFileCancelled,
TotalCancelled: movieCancelled + seriesCancelled + rawFileCancelled,
Queue: status,
})
}
func (h *LibraryHandler) metadataMatchBacklogConfigured() bool {
return h.folderRepo != nil &&
(h.MovieMatchQueueRepo != nil || h.SeriesMatchQueueRepo != nil || h.RawMatchBacklogRepo != nil)
}
func (h *LibraryHandler) metadataMatchQueueStatus(ctx context.Context, libraryID int) (libraryMetadataMatchQueueStatusResponse, error) {
resp := libraryMetadataMatchQueueStatusResponse{LibraryID: libraryID}
if h.MovieMatchQueueRepo != nil {
count, err := h.MovieMatchQueueRepo.CountByFolder(ctx, libraryID)
if err != nil {
return resp, err
}
resp.MovieCount = count
}
if h.SeriesMatchQueueRepo != nil {
count, err := h.SeriesMatchQueueRepo.CountByFolder(ctx, libraryID)
if err != nil {
return resp, err
}
resp.SeriesCount = count
}
if h.RawMatchBacklogRepo != nil {
count, err := h.RawMatchBacklogRepo.CountUnmatchedMatchBacklogByFolder(ctx, libraryID, h.rawMatchBacklogMode())
if err != nil {
return resp, err
}
resp.RawFileCount = count
}
resp.TotalCount = resp.MovieCount + resp.SeriesCount + resp.RawFileCount
return resp, nil
}
func (h *LibraryHandler) rawMatchBacklogMode() scanner.RawMatchBacklogMode {
if h.TVSeriesRootQueue && h.MovieMatchQueueRepo != nil {
return scanner.RawMatchBacklogMixed
}
if h.TVSeriesRootQueue {
return scanner.RawMatchBacklogNonSeries
}
return scanner.RawMatchBacklogGeneric
}
// HandleRefreshLibraryMetadata handles POST /libraries/{id}/refresh-metadata.
// It queues a background admin job to refresh metadata for items in the
// specified library. Quick mode is the default.
@@ -1643,8 +2036,10 @@ func (h *LibraryHandler) seedDefaultChain(ctx context.Context, libraryType strin
levels = []string{"series", "season", "episode"}
case "movies", "movie":
levels = []string{"movie"}
case "audiobooks", "audiobook":
levels = []string{"audiobook"}
case "mixed":
levels = []string{"movie", "series", "season", "episode"}
levels = []string{"movie", "series", "season", "episode", "audiobook"}
default:
return nil
}
internal/api/handlers/libraries_match_test.go
+76
View File
@@ -1,12 +1,15 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/go-chi/chi/v5"
"github.com/Silo-Server/silo-server/internal/models"
)
// TestLibraryMatchUnmatchedItems_NilPool verifies that the unmatched-items
@@ -166,6 +169,79 @@ func TestLibraryMatchRematch_DeprecatedStillRoutes(t *testing.T) {
}
}
func TestLibraryMetadataMatchQueueHandlers_NilFolderRepo(t *testing.T) {
h := &LibraryHandler{MovieMatchQueueRepo: noopMovieMatchQueue{}}
tests := []struct {
name string
method string
path string
handler http.HandlerFunc
}{
{
name: "get",
method: http.MethodGet,
path: "/libraries/1/metadata-match-queue",
handler: h.HandleGetMetadataMatchQueue,
},
{
name: "retry",
method: http.MethodPost,
path: "/libraries/1/metadata-match-queue/retry",
handler: h.HandleRetryMetadataMatchQueue,
},
{
name: "cancel",
method: http.MethodDelete,
path: "/libraries/1/metadata-match-queue",
handler: h.HandleCancelMetadataMatchQueue,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := chi.NewRouter()
r.Method(tt.method, "/libraries/{id}/metadata-match-queue", tt.handler)
r.Method(tt.method, "/libraries/{id}/metadata-match-queue/retry", tt.handler)
req := httptest.NewRequest(tt.method, tt.path, nil)
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("expected 503 for nil folderRepo, got %d: %s", rec.Code, rec.Body.String())
}
var resp errorResponse
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
t.Fatalf("decoding error response: %v", err)
}
if resp.Error != "unavailable" {
t.Errorf("expected error code 'unavailable', got %q", resp.Error)
}
})
}
}
type noopMovieMatchQueue struct{}
func (noopMovieMatchQueue) SyncForFolder(context.Context, int) error {
return nil
}
func (noopMovieMatchQueue) DeleteByFolder(context.Context, int) (int, error) {
return 0, nil
}
func (noopMovieMatchQueue) CountByFolder(context.Context, int) (int, error) {
return 0, nil
}
func (noopMovieMatchQueue) ListByFolder(context.Context, int, int, int) ([]models.MovieMatchQueueEntry, int, error) {
return nil, 0, nil
}
func keys(m map[string]any) []string {
out := make([]string, 0, len(m))
for k := range m {
internal/api/handlers/library_collections.go
+7
View File
@@ -724,6 +724,13 @@ func templateEligibleForLibrary(tmpl templates.Template, library *models.MediaFo
return tmpl.MediaKind == templates.MediaMovie || tmpl.MediaKind == templates.MediaMixed
case "series", "tv", "show", "shows", "tvshows":
return tmpl.MediaKind == templates.MediaTV || tmpl.MediaKind == templates.MediaMixed
case "audiobook", "audiobooks":
// Audiobook libraries only accept audiobook-targeted templates.
// Today the catalog has none (built-ins are TMDB/Trakt/MDBList
// movie/TV imports), so this evaluates to an empty gallery —
// the correct UX. Previously this fell through to default:true
// and offered admins broken movie/TV templates.
return tmpl.MediaKind == templates.MediaAudiobook || tmpl.MediaKind == templates.MediaMixed
default:
return true
}
internal/api/handlers/markers.go
+849
View File
@@ -0,0 +1,849 @@
package handlers
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
chimw "github.com/go-chi/chi/v5/middleware"
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/clientip"
"github.com/Silo-Server/silo-server/internal/markers"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/scanner"
)
// MarkerFileResolver loads a media file by id for the manual-marker API.
type MarkerFileResolver interface {
GetByID(ctx context.Context, id int) (*models.MediaFile, error)
GetByContentID(ctx context.Context, contentID string) ([]*models.MediaFile, error)
GetByEpisodeID(ctx context.Context, episodeID string) ([]*models.MediaFile, error)
}
// ManualMarkerWriter persists/clears manual markers.
type ManualMarkerWriter interface {
UpsertMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate) (bool, error)
ClearMarkers(ctx context.Context, fileID int, segments []string) (bool, error)
UpsertAndClearMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate, clearSegments []string) (bool, error)
}
// MarkerContributor submits a file's eligible markers to enabled providers.
type MarkerContributor interface {
ContributeFile(ctx context.Context, file *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error)
}
// MarkerContributionLister reads contribution history for a file.
type MarkerContributionLister interface {
ListByFile(ctx context.Context, fileID int) ([]markers.ContributionRow, error)
}
type MarkerAuditLister interface {
ListMarkerEditAudit(ctx context.Context, fileIDs []int, limit int) ([]scanner.MarkerEditAuditRow, error)
ListAllMarkerEditAudit(ctx context.Context, limit int) ([]scanner.MarkerEditAuditRow, error)
}
type MarkerPermissionUserLoader interface {
GetByID(ctx context.Context, id int) (*models.User, error)
}
// MarkersHandler serves the manual-marker + contribution API. The marker
// read/write/clear routes are mounted for any authenticated viewer (users fix
// and create markers from the player); the contribution + history routes stay
// admin-only. A successful manual write fires a background contribution run
// (see maybeContribute) so corrected markers reach enabled providers.
type MarkersHandler struct {
Files MarkerFileResolver
Writer ManualMarkerWriter
Contributor MarkerContributor
Contributions MarkerContributionLister
AuditHistory MarkerAuditLister
Notifier PlaybackMarkerUpdateNotifier
Users MarkerPermissionUserLoader
// Authorizer enforces per-item access on file lookups so a viewer can only
// edit markers for content they can actually watch. When nil (tests) the
// handler falls back to an unchecked lookup.
Authorizer *MediaFileAuthorizer
// BaseContext is the lifetime context for detached background work (set from
// the app context) so contribution goroutines cancel on shutdown.
BaseContext context.Context
logger *slog.Logger
}
// NewMarkersHandler constructs the handler.
func NewMarkersHandler(files MarkerFileResolver, writer ManualMarkerWriter, contributor MarkerContributor, contributions MarkerContributionLister, notifier PlaybackMarkerUpdateNotifier, logger *slog.Logger) *MarkersHandler {
if logger == nil {
logger = slog.Default()
}
return &MarkersHandler{Files: files, Writer: writer, Contributor: contributor, Contributions: contributions, Notifier: notifier, logger: logger}
}
var manualMarkerConfidence = 1.0
const manualMarkerAlgorithm = "manual:v1"
// markerContributeTimeout bounds the detached contribution run kicked off after
// a manual save so a slow or rate-limited provider can't leak goroutines.
const markerContributeTimeout = 30 * time.Second
var markerSegmentNames = []string{"intro", "credits", "recap", "preview"}
type segmentInput struct {
Start *float64 `json:"start"`
End *float64 `json:"end"`
}
type segmentMarker struct {
Start *float64 `json:"start"`
End *float64 `json:"end"`
Source *string `json:"source"`
Provider *string `json:"provider"`
Confidence *float64 `json:"confidence"`
Algorithm *string `json:"algorithm"`
DetectedAt *time.Time `json:"detected_at"`
}
type fileMarkersResponse struct {
FileID int `json:"file_id"`
Intro segmentMarker `json:"intro"`
Credits segmentMarker `json:"credits"`
Recap segmentMarker `json:"recap"`
Preview segmentMarker `json:"preview"`
}
type contributionOutcomeResponse struct {
Provider string `json:"provider"`
Segment string `json:"segment"`
Status string `json:"status"`
SubmissionID string `json:"submission_id,omitempty"`
Reason string `json:"reason,omitempty"`
RetryAfterSeconds int `json:"retry_after_seconds,omitempty"`
}
type contributionRowResponse struct {
ID string `json:"id"`
MediaFileID int `json:"media_file_id"`
Provider string `json:"provider"`
Segment string `json:"segment"`
Source string `json:"source"`
SubmittedStartMS *int64 `json:"submitted_start_ms,omitempty"`
SubmittedEndMS *int64 `json:"submitted_end_ms,omitempty"`
VideoDurationMS *int64 `json:"video_duration_ms,omitempty"`
ContentHash string `json:"content_hash"`
SubmissionID *string `json:"submission_id,omitempty"`
Status string `json:"status"`
HTTPStatus *int `json:"http_status,omitempty"`
Error *string `json:"error,omitempty"`
SubmittedAt *time.Time `json:"submitted_at,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
}
type markerEditAuditResponse struct {
ID int64 `json:"id"`
MediaFileID int `json:"media_file_id"`
ItemID *string `json:"item_id,omitempty"`
ItemType *string `json:"item_type,omitempty"`
MediaTitle *string `json:"media_title,omitempty"`
FilePath *string `json:"file_path,omitempty"`
Segment string `json:"segment"`
Action string `json:"action"`
Before *segmentMarker `json:"before"`
After *segmentMarker `json:"after"`
UserID *int `json:"user_id,omitempty"`
Username *string `json:"username,omitempty"`
ImpersonatorUserID *int `json:"impersonator_user_id,omitempty"`
ImpersonatorUsername *string `json:"impersonator_username,omitempty"`
APIKeyID *int64 `json:"api_key_id,omitempty"`
RequestID *string `json:"request_id,omitempty"`
ClientIP *string `json:"client_ip,omitempty"`
UserAgent *string `json:"user_agent,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
func fileMarkers(file *models.MediaFile) fileMarkersResponse {
return fileMarkersResponse{
FileID: file.ID,
Intro: segmentMarker{file.IntroStart, file.IntroEnd, file.IntroMarkersSource, file.IntroMarkersProvider,
file.IntroMarkersConfidence, file.IntroMarkersAlgorithm, file.IntroMarkersDetectedAt},
Credits: segmentMarker{file.CreditsStart, file.CreditsEnd, file.CreditsMarkersSource, file.CreditsMarkersProvider,
file.CreditsMarkersConfidence, file.CreditsMarkersAlgorithm, file.CreditsMarkersDetectedAt},
Recap: segmentMarker{file.RecapStart, file.RecapEnd, file.RecapMarkersSource, file.RecapMarkersProvider,
file.RecapMarkersConfidence, file.RecapMarkersAlgorithm, file.RecapMarkersDetectedAt},
Preview: segmentMarker{file.PreviewStart, file.PreviewEnd, file.PreviewMarkersSource, file.PreviewMarkersProvider,
file.PreviewMarkersConfidence, file.PreviewMarkersAlgorithm, file.PreviewMarkersDetectedAt},
}
}
func (h *MarkersHandler) loadFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
if h == nil || h.Files == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
return nil, false
}
fileID, err := strconv.Atoi(chi.URLParam(r, "fileId"))
if err != nil || fileID <= 0 {
writeError(w, http.StatusBadRequest, "bad_request", "A valid file id is required")
return nil, false
}
return h.authorizeFile(w, r, fileID)
}
// authorizeFile loads a file, enforcing per-item access when an authorizer is
// configured. The 404-on-denial mirrors the playback/subtitle paths so marker
// editing can't be used to probe content outside a profile's library scope.
func (h *MarkersHandler) authorizeFile(w http.ResponseWriter, r *http.Request, fileID int) (*models.MediaFile, bool) {
if h.Authorizer != nil {
file, err := h.Authorizer.Authorize(r, fileID)
if err != nil {
switch {
case errors.Is(err, catalog.ErrItemNotFound), errors.Is(err, catalog.ErrEpisodeNotFound):
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
default:
h.logger.Error("markers: authorize failed", "file_id", fileID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
}
return nil, false
}
return file, true
}
file, err := h.Files.GetByID(r.Context(), fileID)
if err != nil || file == nil {
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
return nil, false
}
return file, true
}
func (h *MarkersHandler) loadItemPrimaryFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
if h == nil || h.Files == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
return nil, false
}
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
if itemID == "" {
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
return nil, false
}
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
if err != nil {
h.logger.Error("markers: episode file lookup failed", "item_id", itemID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
return nil, false
}
if len(files) == 0 {
files, err = h.Files.GetByContentID(r.Context(), itemID)
if err != nil {
h.logger.Error("markers: content file lookup failed", "item_id", itemID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
return nil, false
}
}
if len(files) == 0 {
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
return nil, false
}
// Re-validate the resolved primary file through the authorizer so item-id
// edits are access-checked the same way file-id edits are.
if h.Authorizer != nil {
for _, file := range files {
if file == nil {
continue
}
authorized, err := h.Authorizer.Authorize(r, file.ID)
if err == nil {
return authorized, true
}
if errors.Is(err, catalog.ErrItemNotFound) || errors.Is(err, catalog.ErrEpisodeNotFound) {
continue
}
h.logger.Error("markers: authorize item file failed", "item_id", itemID, "file_id", file.ID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
return nil, false
}
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
return nil, false
}
for _, file := range files {
if file != nil {
return file, true
}
}
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
return nil, false
}
func (h *MarkersHandler) requireMarkerEdit(w http.ResponseWriter, r *http.Request) bool {
claims := apimw.GetClaims(r.Context())
if claims == nil {
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
return false
}
if claims.Role == "admin" {
return true
}
if h == nil || h.Users == nil {
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
return false
}
user, err := h.Users.GetByID(r.Context(), claims.UserID)
if err != nil || user == nil || !auth.HasEffectivePermission(user, auth.PermissionMarkerEdit) {
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
return false
}
return true
}
func (h *MarkersHandler) auditContext(r *http.Request) context.Context {
claims := apimw.GetClaims(r.Context())
if claims == nil {
return r.Context()
}
audit := scanner.MarkerAuditContext{
UserID: &claims.UserID,
ImpersonatorUserID: claims.ImpersonatorUserID,
RequestID: chimw.GetReqID(r.Context()),
ClientIP: clientip.FromContext(r.Context()),
UserAgent: r.UserAgent(),
}
if claims.APIKeyID > 0 {
apiKeyID := claims.APIKeyID
audit.APIKeyID = &apiKeyID
}
return scanner.WithMarkerAuditContext(r.Context(), audit)
}
// HandleGetFileMarkers returns the current markers + provenance for a file.
func (h *MarkersHandler) HandleGetFileMarkers(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadFile(w, r)
if !ok {
return
}
writeJSON(w, http.StatusOK, fileMarkers(file))
}
// HandleGetItemMarkers returns markers for the item's primary file.
func (h *MarkersHandler) HandleGetItemMarkers(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadItemPrimaryFile(w, r)
if !ok {
return
}
writeJSON(w, http.StatusOK, fileMarkers(file))
}
// HandleSetFileMarkers upserts the manual marker layer. Each segment key may be
// an object {start, end} to set, or null to clear; absent keys are unchanged.
func (h *MarkersHandler) HandleSetFileMarkers(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadFile(w, r)
if !ok {
return
}
if !h.requireMarkerEdit(w, r) {
return
}
h.setMarkersForFile(w, r, file)
}
// HandleSetItemMarkers upserts manual markers on the item's primary file.
func (h *MarkersHandler) HandleSetItemMarkers(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadItemPrimaryFile(w, r)
if !ok {
return
}
if !h.requireMarkerEdit(w, r) {
return
}
h.setMarkersForFile(w, r, file)
}
func (h *MarkersHandler) setMarkersForFile(w http.ResponseWriter, r *http.Request, file *models.MediaFile) {
if h.Writer == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
return
}
var raw map[string]json.RawMessage
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
duration := float64(file.Duration)
update := scanner.MarkerUpdate{
MarkersSource: models.MarkerSourceManual,
MarkersConfidence: &manualMarkerConfidence,
MarkersAlgorithm: manualMarkerAlgorithm,
}
var clears []string
var setSegs []string
for _, seg := range markerSegmentNames {
val, present := raw[seg]
if !present {
continue
}
if isJSONNull(val) {
clears = append(clears, seg)
continue
}
var in segmentInput
if err := json.Unmarshal(val, &in); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid "+seg+" marker")
return
}
start, end, err := normalizeManualSegment(seg, in, duration)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
applyManualSegment(&update, seg, start, end)
setSegs = append(setSegs, seg)
}
if _, err := h.Writer.UpsertAndClearMarkers(h.auditContext(r), file.ID, update, clears); err != nil {
h.logger.Error("markers: save failed", "file_id", file.ID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save markers")
return
}
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
if err != nil {
h.logger.Error("markers: reload after save failed", "file_id", file.ID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Markers saved but failed to reload")
return
}
h.maybeContribute(refreshed, setSegs)
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
}
// HandleClearFileSegment clears a single segment.
func (h *MarkersHandler) HandleClearFileSegment(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadFile(w, r)
if !ok {
return
}
if h.Writer == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
return
}
if !h.requireMarkerEdit(w, r) {
return
}
segment := chi.URLParam(r, "segment")
if !isMarkerSegment(segment) {
writeError(w, http.StatusBadRequest, "bad_request", "Unknown marker segment")
return
}
if _, err := h.Writer.ClearMarkers(h.auditContext(r), file.ID, []string{segment}); err != nil {
h.logger.Error("markers: clear segment failed", "file_id", file.ID, "segment", segment, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to clear marker")
return
}
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
if err != nil {
h.logger.Error("markers: reload after clear failed", "file_id", file.ID, "segment", segment, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Marker cleared but failed to reload")
return
}
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
}
// HandleContributeFile submits the file's eligible markers to enabled providers.
func (h *MarkersHandler) HandleContributeFile(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadFile(w, r)
if !ok {
return
}
if h.Contributor == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Contribution is not configured")
return
}
var body struct {
Provider string `json:"provider"`
Segments []string `json:"segments"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
var kinds []markers.MarkerKind
for _, name := range body.Segments {
kind, ok := markerKindForName(name)
if !ok {
writeError(w, http.StatusBadRequest, "bad_request", "Unknown segment "+name)
return
}
kinds = append(kinds, kind)
}
outcomes, err := h.Contributor.ContributeFile(r.Context(), file, markers.ContributeOptions{Provider: body.Provider, Segments: kinds})
if err != nil {
h.logger.Error("markers: contribute failed", "file_id", file.ID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Contribution failed")
return
}
writeJSON(w, http.StatusOK, map[string]any{"outcomes": contributionOutcomeResponses(outcomes)})
}
// HandleListFileContributions returns the contribution history for a file.
func (h *MarkersHandler) HandleListFileContributions(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadFile(w, r)
if !ok {
return
}
if h.Contributions == nil {
writeJSON(w, http.StatusOK, map[string]any{"contributions": []contributionRowResponse{}})
return
}
rows, err := h.Contributions.ListByFile(r.Context(), file.ID)
if err != nil {
h.logger.Error("markers: list contributions failed", "file_id", file.ID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load contributions")
return
}
writeJSON(w, http.StatusOK, map[string]any{"contributions": contributionRowResponses(rows)})
}
// HandleListFileMarkerHistory returns recent semantic marker edit audit rows for one file.
func (h *MarkersHandler) HandleListFileMarkerHistory(w http.ResponseWriter, r *http.Request) {
file, ok := h.loadFile(w, r)
if !ok {
return
}
h.listMarkerHistory(w, r, []int{file.ID})
}
// HandleListItemMarkerHistory returns recent marker edit audit rows for every file version on an item.
func (h *MarkersHandler) HandleListItemMarkerHistory(w http.ResponseWriter, r *http.Request) {
if h == nil || h.Files == nil {
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker history is not configured")
return
}
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
if itemID == "" {
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
return
}
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
if err != nil {
h.logger.Error("markers: episode history file lookup failed", "item_id", itemID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
return
}
if len(files) == 0 {
files, err = h.Files.GetByContentID(r.Context(), itemID)
if err != nil {
h.logger.Error("markers: content history file lookup failed", "item_id", itemID, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
return
}
}
fileIDs := make([]int, 0, len(files))
for _, file := range files {
if file != nil {
fileIDs = append(fileIDs, file.ID)
}
}
if len(fileIDs) == 0 {
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
return
}
h.listMarkerHistory(w, r, fileIDs)
}
// HandleListMarkerHistory returns recent semantic marker edit audit rows across all files.
func (h *MarkersHandler) HandleListMarkerHistory(w http.ResponseWriter, r *http.Request) {
if h.AuditHistory == nil {
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
return
}
limit, ok := markerHistoryLimit(w, r)
if !ok {
return
}
rows, err := h.AuditHistory.ListAllMarkerEditAudit(r.Context(), limit)
if err != nil {
h.logger.Error("markers: list all history failed", "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
return
}
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
}
func (h *MarkersHandler) listMarkerHistory(w http.ResponseWriter, r *http.Request, fileIDs []int) {
if h.AuditHistory == nil {
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
return
}
limit, ok := markerHistoryLimit(w, r)
if !ok {
return
}
rows, err := h.AuditHistory.ListMarkerEditAudit(r.Context(), fileIDs, limit)
if err != nil {
h.logger.Error("markers: list history failed", "file_ids", fileIDs, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
return
}
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
}
func markerHistoryLimit(w http.ResponseWriter, r *http.Request) (int, bool) {
limit := 25
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
parsed, err := strconv.Atoi(raw)
if err != nil || parsed <= 0 {
writeError(w, http.StatusBadRequest, "bad_request", "A valid limit is required")
return 0, false
}
limit = parsed
}
return limit, true
}
func (h *MarkersHandler) reloadAndNotify(ctx context.Context, fileID int) (*models.MediaFile, error) {
refreshed, err := h.Files.GetByID(ctx, fileID)
if err != nil || refreshed == nil {
if err == nil {
err = scanner.ErrFileNotFound
}
return nil, err
}
if h.Notifier != nil {
h.Notifier.MarkersUpdated(ctx, refreshed)
}
return refreshed, nil
}
// maybeContribute fires a best-effort, detached contribution run for the
// segments just set by a manual save. ContributeFile self-gates on each
// provider's contribute_enabled flag, only submits scanner/manual-sourced
// segments, and is idempotent — so this is a no-op when contribution is off or
// the marker was already submitted. It runs in the background so the save
// responds immediately and is never blocked by provider rate limits.
func (h *MarkersHandler) maybeContribute(file *models.MediaFile, segments []string) {
if h == nil || h.Contributor == nil || file == nil || len(segments) == 0 {
return
}
kinds := make([]markers.MarkerKind, 0, len(segments))
for _, seg := range segments {
if kind, ok := markerKindForName(seg); ok {
kinds = append(kinds, kind)
}
}
if len(kinds) == 0 {
return
}
base := h.BaseContext
if base == nil {
base = context.Background()
}
go func() {
ctx, cancel := context.WithTimeout(base, markerContributeTimeout)
defer cancel()
if _, err := h.Contributor.ContributeFile(ctx, file, markers.ContributeOptions{Segments: kinds}); err != nil {
h.logger.Warn("markers: background contribution failed", "file_id", file.ID, "error", err)
}
}()
}
func isJSONNull(raw json.RawMessage) bool {
return string(bytes.TrimSpace(raw)) == "null"
}
func isMarkerSegment(seg string) bool {
_, ok := markerKindForName(seg)
return ok
}
func markerKindForName(name string) (markers.MarkerKind, bool) {
switch name {
case "intro":
return markers.MarkerKindIntro, true
case "credits":
return markers.MarkerKindCredits, true
case "recap":
return markers.MarkerKindRecap, true
case "preview":
return markers.MarkerKindPreview, true
default:
return 0, false
}
}
func markerNameForKind(kind markers.MarkerKind) string {
switch kind {
case markers.MarkerKindIntro:
return "intro"
case markers.MarkerKindCredits:
return "credits"
case markers.MarkerKindRecap:
return "recap"
case markers.MarkerKindPreview:
return "preview"
default:
return ""
}
}
func contributionOutcomeResponses(outcomes []markers.ContributionOutcome) []contributionOutcomeResponse {
resp := make([]contributionOutcomeResponse, 0, len(outcomes))
for _, o := range outcomes {
item := contributionOutcomeResponse{
Provider: o.Provider,
Segment: markerNameForKind(o.Segment),
Status: o.Status,
SubmissionID: o.SubmissionID,
Reason: o.Reason,
}
if o.RetryAfter > 0 {
item.RetryAfterSeconds = int(o.RetryAfter.Seconds())
}
resp = append(resp, item)
}
return resp
}
func contributionRowResponses(rows []markers.ContributionRow) []contributionRowResponse {
resp := make([]contributionRowResponse, 0, len(rows))
for _, row := range rows {
item := contributionRowResponse{
ID: row.ID,
MediaFileID: row.MediaFileID,
Provider: row.Provider,
Segment: row.SegmentKind,
Source: row.Source,
SubmittedStartMS: row.SubmittedStartMs,
SubmittedEndMS: row.SubmittedEndMs,
VideoDurationMS: row.VideoDurationMs,
ContentHash: row.ContentHash,
SubmissionID: row.SubmissionID,
Status: row.Status,
HTTPStatus: row.HTTPStatus,
Error: row.Error,
}
if !row.SubmittedAt.IsZero() {
submittedAt := row.SubmittedAt
item.SubmittedAt = &submittedAt
}
if !row.UpdatedAt.IsZero() {
updatedAt := row.UpdatedAt
item.UpdatedAt = &updatedAt
}
resp = append(resp, item)
}
return resp
}
func markerEditAuditResponses(rows []scanner.MarkerEditAuditRow) []markerEditAuditResponse {
resp := make([]markerEditAuditResponse, 0, len(rows))
for _, row := range rows {
resp = append(resp, markerEditAuditResponse{
ID: row.ID,
MediaFileID: row.MediaFileID,
ItemID: row.ItemID,
ItemType: row.ItemType,
MediaTitle: row.MediaTitle,
FilePath: row.FilePath,
Segment: row.SegmentKind,
Action: row.Action,
Before: markerAuditSegmentResponse(row.Before),
After: markerAuditSegmentResponse(row.After),
UserID: row.UserID,
Username: row.Username,
ImpersonatorUserID: row.ImpersonatorUserID,
ImpersonatorUsername: row.ImpersonatorUsername,
APIKeyID: row.APIKeyID,
RequestID: row.RequestID,
ClientIP: row.ClientIP,
UserAgent: row.UserAgent,
CreatedAt: row.CreatedAt,
})
}
return resp
}
func markerAuditSegmentResponse(segment *scanner.MarkerAuditSegment) *segmentMarker {
if segment == nil {
return nil
}
return &segmentMarker{
Start: segment.Start,
End: segment.End,
Source: segment.Source,
Provider: segment.Provider,
Confidence: segment.Confidence,
Algorithm: segment.Algorithm,
DetectedAt: segment.DetectedAt,
}
}
// normalizeManualSegment applies the start/end defaults and validation, mirroring
// the contribution rules: intro/recap may omit start (=0); credits/preview may
// omit end (=duration); end must exceed start and stay within the file.
func normalizeManualSegment(seg string, in segmentInput, duration float64) (start, end float64, err error) {
switch seg {
case "intro", "recap":
if in.Start != nil {
start = *in.Start
}
if in.End == nil {
return 0, 0, errSegment(seg, "end is required")
}
end = *in.End
default: // credits, preview
if in.Start == nil {
return 0, 0, errSegment(seg, "start is required")
}
start = *in.Start
if in.End != nil {
end = *in.End
} else if duration > 0 {
end = duration
} else {
return 0, 0, errSegment(seg, "end is required when duration is unknown")
}
}
if start < 0 || end <= start {
return 0, 0, errSegment(seg, "end must be greater than start")
}
if duration > 0 && end > duration+1 {
return 0, 0, errSegment(seg, "end exceeds the file duration")
}
return start, end, nil
}
func applyManualSegment(update *scanner.MarkerUpdate, seg string, start, end float64) {
s, e := start, end
switch seg {
case "intro":
update.IntroStart, update.IntroEnd = &s, &e
case "credits":
update.CreditsStart, update.CreditsEnd = &s, &e
case "recap":
update.RecapStart, update.RecapEnd = &s, &e
case "preview":
update.PreviewStart, update.PreviewEnd = &s, &e
}
}
func errSegment(seg, msg string) error {
return &markerValidationError{seg: seg, msg: msg}
}
type markerValidationError struct {
seg string
msg string
}
func (e *markerValidationError) Error() string { return e.seg + " marker: " + e.msg }
internal/api/handlers/markers_test.go
+524
View File
@@ -0,0 +1,524 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/go-chi/chi/v5"
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/clientip"
"github.com/Silo-Server/silo-server/internal/markers"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/scanner"
)
type fakeMarkerFiles struct {
file *models.MediaFile
byID map[int]*models.MediaFile
contentFiles []*models.MediaFile
episodeFiles []*models.MediaFile
}
func (f fakeMarkerFiles) GetByID(_ context.Context, id int) (*models.MediaFile, error) {
if f.byID != nil {
return f.byID[id], nil
}
return f.file, nil
}
func (f fakeMarkerFiles) GetByContentID(context.Context, string) ([]*models.MediaFile, error) {
return f.contentFiles, nil
}
func (f fakeMarkerFiles) GetByEpisodeID(context.Context, string) ([]*models.MediaFile, error) {
return f.episodeFiles, nil
}
type fakeMarkerWriter struct {
upserts []scanner.MarkerUpdate
clears [][]string
audits []scanner.MarkerAuditContext
}
func (f *fakeMarkerWriter) captureAudit(ctx context.Context) {
if audit, ok := scanner.MarkerAuditContextFromContext(ctx); ok {
f.audits = append(f.audits, audit)
}
}
func (f *fakeMarkerWriter) UpsertMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate) (bool, error) {
f.captureAudit(ctx)
f.upserts = append(f.upserts, u)
return true, nil
}
func (f *fakeMarkerWriter) ClearMarkers(ctx context.Context, _ int, segs []string) (bool, error) {
f.captureAudit(ctx)
f.clears = append(f.clears, segs)
return true, nil
}
func (f *fakeMarkerWriter) UpsertAndClearMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate, segs []string) (bool, error) {
f.captureAudit(ctx)
if u.HasAnySegment() {
f.upserts = append(f.upserts, u)
}
if len(segs) > 0 {
f.clears = append(f.clears, segs)
}
return true, nil
}
func markerPutRequest(body string) *http.Request {
req := httptest.NewRequest(http.MethodPut, "/markers/files/5", strings.NewReader(body))
rctx := chi.NewRouteContext()
rctx.URLParams.Add("fileId", "5")
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
}
func markerItemPutRequest(body string) *http.Request {
req := httptest.NewRequest(http.MethodPut, "/markers/items/episode-1", strings.NewReader(body))
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", "episode-1")
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
}
func withMarkerAdminClaims(req *http.Request) *http.Request {
return req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{
UserID: 1,
Role: "admin",
TokenType: auth.TokenTypeAccess,
SessionID: "session-1",
}))
}
func newMarkersHandler(writer ManualMarkerWriter) *MarkersHandler {
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
return NewMarkersHandler(files, writer, nil, nil, nil, nil)
}
func TestSetFileMarkersWritesManual(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
if len(writer.upserts) != 1 {
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
}
u := writer.upserts[0]
if u.MarkersSource != models.MarkerSourceManual {
t.Errorf("source = %q, want manual", u.MarkersSource)
}
if u.IntroStart == nil || *u.IntroStart != 0 || u.IntroEnd == nil || *u.IntroEnd != 60 {
t.Errorf("intro = %v..%v, want 0..60", u.IntroStart, u.IntroEnd)
}
}
func TestSetItemMarkersWritesPrimaryEpisodeFile(t *testing.T) {
writer := &fakeMarkerWriter{}
file := &models.MediaFile{ID: 8, Duration: 1800}
files := fakeMarkerFiles{
byID: map[int]*models.MediaFile{8: file},
episodeFiles: []*models.MediaFile{file},
}
h := NewMarkersHandler(files, writer, nil, nil, nil, nil)
rec := httptest.NewRecorder()
h.HandleSetItemMarkers(rec, markerItemPutRequest(`{"recap":{"start":0,"end":45}}`))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
if len(writer.upserts) != 1 {
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
}
if writer.upserts[0].RecapEnd == nil || *writer.upserts[0].RecapEnd != 45 {
t.Errorf("recap end = %v, want 45", writer.upserts[0].RecapEnd)
}
}
type fakeMarkerItemAccess map[string]error
func (f fakeMarkerItemAccess) EnsureAccessible(_ context.Context, contentID string, _ catalog.AccessFilter) error {
return f[contentID]
}
type fakeMarkerEpisodeLookup map[string]*models.Episode
func (f fakeMarkerEpisodeLookup) GetByID(_ context.Context, contentID string) (*models.Episode, error) {
return f[contentID], nil
}
type fakeMarkerUsers map[int]*models.User
func (f fakeMarkerUsers) GetByID(_ context.Context, id int) (*models.User, error) {
return f[id], nil
}
func TestGetItemMarkersUsesFirstAuthorizedFile(t *testing.T) {
denied := &models.MediaFile{ID: 8, EpisodeID: "episode-denied", Duration: 1800}
allowed := &models.MediaFile{ID: 9, EpisodeID: "episode-allowed", Duration: 1800}
files := fakeMarkerFiles{
byID: map[int]*models.MediaFile{
8: denied,
9: allowed,
},
episodeFiles: []*models.MediaFile{denied, allowed},
}
h := NewMarkersHandler(files, nil, nil, nil, nil, nil)
h.Authorizer = &MediaFileAuthorizer{
FileResolver: files,
ItemAccess: fakeMarkerItemAccess{
"series-denied": catalog.ErrItemNotFound,
},
EpisodeLookup: fakeMarkerEpisodeLookup{
"episode-denied": {ContentID: "episode-denied", SeriesID: "series-denied"},
"episode-allowed": {ContentID: "episode-allowed", SeriesID: "series-allowed"},
},
}
req := httptest.NewRequest(http.MethodGet, "/markers/items/episode-1", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", "episode-1")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rec := httptest.NewRecorder()
h.HandleGetItemMarkers(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
var body fileMarkersResponse
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("decode response: %v", err)
}
if body.FileID != allowed.ID {
t.Fatalf("file_id = %d, want first authorized file %d", body.FileID, allowed.ID)
}
}
func TestGetFileMarkersDoesNotRequireMarkerEditPermission(t *testing.T) {
h := newMarkersHandler(nil)
h.Users = fakeMarkerUsers{
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
}
req := httptest.NewRequest(http.MethodGet, "/markers/files/5", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("fileId", "5")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
rec := httptest.NewRecorder()
h.HandleGetFileMarkers(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
}
func TestSetFileMarkersRejectsUserWithoutMarkerEditPermission(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
h.Users = fakeMarkerUsers{
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
}
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403; body=%s", rec.Code, rec.Body.String())
}
if len(writer.upserts) != 0 {
t.Fatalf("permission failure wrote markers: %+v", writer.upserts)
}
}
func TestSetFileMarkersAllowsUserWithMarkerEditPermission(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
h.Users = fakeMarkerUsers{
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: []string{"marker_edit"}},
}
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
if len(writer.upserts) != 1 {
t.Fatalf("expected marker write, got %d", len(writer.upserts))
}
}
func TestSetFileMarkersPassesAuditContextToWriter(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
apiKeyID := int64(99)
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
req.Header.Set("User-Agent", "marker-test-agent")
ctx := clientip.SetContext(req.Context(), "203.0.113.10")
ctx = apimw.SetClaims(ctx, &auth.Claims{
UserID: 7,
Role: "admin",
TokenType: auth.TokenTypeAPIKey,
APIKeyID: apiKeyID,
})
req = req.WithContext(ctx)
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
if len(writer.audits) != 1 {
t.Fatalf("audit contexts = %d, want 1", len(writer.audits))
}
audit := writer.audits[0]
if audit.UserID == nil || *audit.UserID != 7 {
t.Fatalf("audit user id = %v, want 7", audit.UserID)
}
if audit.APIKeyID == nil || *audit.APIKeyID != apiKeyID {
t.Fatalf("audit api key id = %v, want %d", audit.APIKeyID, apiKeyID)
}
if audit.ClientIP != "203.0.113.10" || audit.UserAgent != "marker-test-agent" {
t.Fatalf("audit request metadata = ip %q ua %q", audit.ClientIP, audit.UserAgent)
}
}
func TestSetFileMarkersClearsOnNull(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
if len(writer.upserts) != 0 {
t.Errorf("expected no upsert for a null clear, got %d", len(writer.upserts))
}
if len(writer.clears) != 1 || len(writer.clears[0]) != 1 || writer.clears[0][0] != "credits" {
t.Errorf("clears = %v, want [[credits]]", writer.clears)
}
}
func TestSetFileMarkersRejectsInvalidRange(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":60,"end":10}}`))
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", rec.Code)
}
if len(writer.upserts) != 0 {
t.Errorf("invalid range must not write, got %d upserts", len(writer.upserts))
}
}
func TestClearFileSegmentRejectsUnknown(t *testing.T) {
writer := &fakeMarkerWriter{}
h := newMarkersHandler(writer)
req := httptest.NewRequest(http.MethodDelete, "/markers/files/5/bogus", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("fileId", "5")
rctx.URLParams.Add("segment", "bogus")
req = withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
rec := httptest.NewRecorder()
h.HandleClearFileSegment(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", rec.Code)
}
}
type fakeMarkerContributor struct{ outcomes []markers.ContributionOutcome }
func (f fakeMarkerContributor) ContributeFile(context.Context, *models.MediaFile, markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
return f.outcomes, nil
}
type captureMarkerContributor struct {
opts markers.ContributeOptions
}
func (c *captureMarkerContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
c.opts = opts
return nil, nil
}
// signalContributor reports every ContributeFile call on a channel so a test
// can wait for the detached background contribution kicked off by a save.
type signalContributor struct {
called chan markers.ContributeOptions
}
func (s signalContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
s.called <- opts
return nil, nil
}
func TestSetFileMarkersTriggersBackgroundContribution(t *testing.T) {
called := make(chan markers.ContributeOptions, 1)
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
select {
case opts := <-called:
if len(opts.Segments) != 1 || opts.Segments[0] != markers.MarkerKindIntro {
t.Fatalf("contribute segments = %v, want [intro]", opts.Segments)
}
case <-time.After(2 * time.Second):
t.Fatal("expected a background contribution for the saved intro segment")
}
}
func TestSetFileMarkersClearOnlyDoesNotContribute(t *testing.T) {
called := make(chan markers.ContributeOptions, 1)
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
rec := httptest.NewRecorder()
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
select {
case <-called:
t.Fatal("a clear-only save must not trigger contribution")
case <-time.After(200 * time.Millisecond):
}
}
type fakeContributionLister struct{ rows []markers.ContributionRow }
func (f fakeContributionLister) ListByFile(context.Context, int) ([]markers.ContributionRow, error) {
return f.rows, nil
}
func TestContributeFileUsesSnakeCaseResponse(t *testing.T) {
h := NewMarkersHandler(
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
nil,
fakeMarkerContributor{outcomes: []markers.ContributionOutcome{{
Provider: "introdb", Segment: markers.MarkerKindCredits, Status: markers.OutcomeStatusRateLimited,
Reason: "usage limited", RetryAfter: 30 * time.Second,
}}},
nil,
nil,
nil,
)
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{}`))
rctx := chi.NewRouteContext()
rctx.URLParams.Add("fileId", "5")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rec := httptest.NewRecorder()
h.HandleContributeFile(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
var body struct {
Outcomes []map[string]any `json:"outcomes"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(body.Outcomes) != 1 || body.Outcomes[0]["segment"] != "credits" {
t.Fatalf("outcomes = %+v, want credits segment name", body.Outcomes)
}
if _, ok := body.Outcomes[0]["Segment"]; ok {
t.Fatalf("response leaked internal Segment field: %s", rec.Body.String())
}
if body.Outcomes[0]["retry_after_seconds"] != float64(30) {
t.Fatalf("retry_after_seconds = %v, want 30", body.Outcomes[0]["retry_after_seconds"])
}
}
func TestContributeFileDecodesUnknownLengthBody(t *testing.T) {
contributor := &captureMarkerContributor{}
h := NewMarkersHandler(
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
nil,
contributor,
nil,
nil,
nil,
)
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{"provider":"introdb","segments":["credits"]}`))
req.ContentLength = -1
rctx := chi.NewRouteContext()
rctx.URLParams.Add("fileId", "5")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rec := httptest.NewRecorder()
h.HandleContributeFile(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
if contributor.opts.Provider != "introdb" {
t.Fatalf("provider = %q, want introdb", contributor.opts.Provider)
}
if len(contributor.opts.Segments) != 1 || contributor.opts.Segments[0] != markers.MarkerKindCredits {
t.Fatalf("segments = %+v, want [credits]", contributor.opts.Segments)
}
}
func TestListFileContributionsUsesSnakeCaseResponse(t *testing.T) {
now := time.Unix(100, 0).UTC()
h := NewMarkersHandler(
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
nil,
nil,
fakeContributionLister{rows: []markers.ContributionRow{{
ID: "row1", MediaFileID: 5, Provider: "introdb", SegmentKind: "intro",
Source: "manual", ContentHash: "hash", Status: "pending", SubmittedAt: now, UpdatedAt: now,
}}},
nil,
nil,
)
req := httptest.NewRequest(http.MethodGet, "/admin/files/5/contributions", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("fileId", "5")
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
rec := httptest.NewRecorder()
h.HandleListFileContributions(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
}
body := rec.Body.String()
if !strings.Contains(body, `"media_file_id":5`) || strings.Contains(body, "MediaFileID") {
t.Fatalf("unexpected contribution response shape: %s", body)
}
}
internal/api/handlers/people.go
+26
View File
@@ -39,6 +39,8 @@ var personRefreshRate = ratelimit.Rate{
Burst: 10,
}
const personMetadataStaleAfter = 90 * 24 * time.Hour
// PeopleHandler serves person-related API endpoints.
type PeopleHandler struct {
personRepo peopleRepository
@@ -131,6 +133,8 @@ func (h *PeopleHandler) HandleGetPerson(w http.ResponseWriter, r *http.Request)
return
}
h.enqueuePersonRefreshIfDue(*person)
writeJSON(w, http.StatusOK, h.toResponse(r.Context(), *person))
}
@@ -359,6 +363,28 @@ func (h *PeopleHandler) toResponse(ctx context.Context, p models.Person) personR
return resp
}
func (h *PeopleHandler) enqueuePersonRefreshIfDue(person models.Person) {
if h.refreshQueue == nil || !personHasRefreshableProviderID(person) {
return
}
if personMetadataIncomplete(person) {
return
}
if person.UpdatedAt.Before(time.Now().Add(-personMetadataStaleAfter)) {
h.refreshQueue.Enqueue(person.ID)
}
}
func personHasRefreshableProviderID(person models.Person) bool {
return person.TmdbID != "" || person.ImdbID != "" || person.TvdbID != ""
}
func personMetadataIncomplete(person models.Person) bool {
return person.Bio == "" || person.PhotoPath == "" || person.PhotoPath == "-" || person.BirthDate == nil
}
func parseOptionalPersonDate(raw string) (*time.Time, error) {
if raw == "" {
return nil, nil
internal/api/handlers/playback.go
+50 -5
View File
@@ -305,6 +305,7 @@ type subtitleURL struct {
Forced bool `json:"forced"`
HearingImpaired bool `json:"hearing_impaired"`
URL string `json:"url"`
FontBundleURL string `json:"font_bundle_url,omitempty"`
}
// changeAudioRequest represents the JSON body for PATCH /playback/{session_id}/audio.
@@ -1403,7 +1404,11 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req
if resp.SubtitleURLs[i].Source == "embedded" {
// Pass the ffmpeg-relative subtitle stream index to the proxy.
embeddedIdx := resp.SubtitleURLs[i].Index - embeddedOffset
resp.SubtitleURLs[i].URL = proxyNode.URL + "/stream/subtitles/" + token + "/" + strconv.Itoa(embeddedIdx) + subtitleURLExt(resp.SubtitleURLs[i].Codec)
proxySubtitleURL := proxyNode.URL + "/stream/subtitles/" + token + "/" + strconv.Itoa(embeddedIdx)
resp.SubtitleURLs[i].URL = proxySubtitleURL + subtitleURLExt(resp.SubtitleURLs[i].Codec)
if resp.SubtitleURLs[i].FontBundleURL != "" {
resp.SubtitleURLs[i].FontBundleURL = proxySubtitleURL + "/fonts"
}
}
}
}
@@ -1459,6 +1464,7 @@ func buildSubtitleURLs(sessionID string, file *models.MediaFile, downloaded []su
Forced: track.Forced,
HearingImpaired: track.HearingImpaired,
URL: fmt.Sprintf("/stream/%s/subtitles/%d%s", sessionID, embeddedOffset+i, subtitleURLExt(track.Codec)),
FontBundleURL: subtitleFontBundleURL(sessionID, embeddedOffset+i, track.Codec),
})
}
@@ -1478,6 +1484,13 @@ func buildSubtitleURLs(sessionID string, file *models.MediaFile, downloaded []su
return urls
}
func subtitleFontBundleURL(sessionID string, trackIndex int, codec string) string {
if !playback.IsASS(codec) {
return ""
}
return fmt.Sprintf("/stream/%s/subtitles/%d/fonts", sessionID, trackIndex)
}
func firstNonEmptyString(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
@@ -1531,6 +1544,7 @@ func (h *PlaybackHandler) HandleUpdateProgress(w http.ResponseWriter, r *http.Re
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update progress")
return
}
h.syncSessionsNow(r.Context(), "progress")
// Persist progress to UserStore (best-effort).
if sess, getErr := h.sessionMgr.GetSession(sessionID); getErr == nil {
@@ -1838,6 +1852,7 @@ type transcodeStartState struct {
file *models.MediaFile
session *playback.Session
switchedFileID *int
hwAccel string
}
// finalizeTranscodeStart updates the playback session state after a transcode
@@ -1878,6 +1893,7 @@ func (h *PlaybackHandler) finalizeTranscodeStart(r *http.Request, st transcodeSt
TargetVideoCodec: st.req.TargetCodecVideo,
TargetAudioCodec: st.req.TargetCodecAudio,
TargetBitrateKbps: st.req.TargetBitrateKbps,
TranscodeHWAccel: st.hwAccel,
}); err != nil {
slog.Error("failed to update transcode stream state", "session", st.req.SessionID, "error", err, "playback_session_id", st.req.SessionID)
}
@@ -2052,16 +2068,35 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re
writeError(w, http.StatusBadGateway, "transcode_node_unavailable", "Transcode node is unavailable")
return
}
resp.Body.Close()
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
slog.Error("remote transcode start rejected", "status", resp.StatusCode, "node", tcNode.URL)
writeError(w, http.StatusBadGateway, "transcode_start_failed", "Transcode node rejected the request")
return
}
var nodeResp transcodenode.TranscodeStartResponse
if err := json.NewDecoder(resp.Body).Decode(&nodeResp); err != nil {
slog.Warn("remote transcode start response decode failed",
"error", err,
"node", tcNode.URL,
"session", req.SessionID,
"playback_session_id", req.SessionID,
)
}
effectiveHWAccel := strings.TrimSpace(nodeResp.HWAccel)
if effectiveHWAccel == "" {
effectiveHWAccel = strings.TrimSpace(nodeReq.HWAccel)
}
manifestURL := h.buildProxyManifestURL(req.SessionID, session, tcNode.URL)
h.finalizeTranscodeStart(r, transcodeStartState{req: req, file: file, session: session, switchedFileID: switchedFileID})
h.finalizeTranscodeStart(r, transcodeStartState{
req: req,
file: file,
session: session,
switchedFileID: switchedFileID,
hwAccel: effectiveHWAccel,
})
writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL))
return
}
@@ -2109,7 +2144,13 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re
h.monitorLocalTranscodeExit(req.SessionID, transcodeSession)
manifestURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", req.SessionID)
h.finalizeTranscodeStart(r, transcodeStartState{req: req, file: file, session: session, switchedFileID: switchedFileID})
h.finalizeTranscodeStart(r, transcodeStartState{
req: req,
file: file,
session: session,
switchedFileID: switchedFileID,
hwAccel: transcodeSession.Opts().HWAccel,
})
writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL))
}
@@ -2216,6 +2257,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
"start_segment_number", decision.Progress.StartSegmentNumber,
"last_produced_age_ms", lastProducedAgeMS,
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
"restart_on_timeout", decision.RestartOnTimeout,
"reason", decision.Reason,
"session", sessionID,
"playback_session_id", sessionID,
@@ -2229,6 +2271,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
"start_segment_number", decision.Progress.StartSegmentNumber,
"last_produced_age_ms", lastProducedAgeMS,
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
"restart_on_timeout", decision.RestartOnTimeout,
"reason", decision.Reason,
"session", sessionID,
"playback_session_id", sessionID,
@@ -2243,6 +2286,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
"start_segment_number", decision.Progress.StartSegmentNumber,
"last_produced_age_ms", lastProducedAgeMS,
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
"restart_on_timeout", decision.RestartOnTimeout,
"reason", decision.Reason,
"session", sessionID,
"playback_session_id", sessionID,
@@ -2254,7 +2298,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
// active encode range), either restart at the exact manifest-derived
// timeline position or return 404 for copy-mode segments outside the
// current manifest window.
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) && decision.RestartOnTimeout {
seekSeconds, ok, seekErr := transcodeSession.RestartSeekTarget(segNum)
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
slog.Error("resolve transcode seek target", "error", seekErr, "segment", segmentName, "session", sessionID, "playback_session_id", sessionID)
@@ -2269,6 +2313,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
"start_segment_number", decision.Progress.StartSegmentNumber,
"last_produced_age_ms", lastProducedAgeMS,
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
"restart_on_timeout", decision.RestartOnTimeout,
"reason", decision.Reason,
"seek_seconds", seekSeconds,
"session", sessionID,
internal/api/handlers/playback_lazy_markers.go
+27 -13
View File
@@ -272,7 +272,7 @@ func (h *PlaybackHandler) fetchOnlineMarkersForPlayback(ctx context.Context, fil
EpisodeNumber: ids.EpisodeNumber,
Duration: time.Duration(file.Duration) * time.Second,
}
result, ok, err := h.MarkerRegistry.FetchFirstHit(ctx, req)
result, ok, err := h.MarkerRegistry.FetchMerged(ctx, req)
if err != nil {
return false, err
}
@@ -373,17 +373,31 @@ func hasLocalDetectionMarkers(file *models.MediaFile) bool {
// each other.
func markerUpdateFromPayload(p markers.MarkerUpdatePayload) scanner.MarkerUpdate {
return scanner.MarkerUpdate{
IntroStart: p.IntroStart,
IntroEnd: p.IntroEnd,
CreditsStart: p.CreditsStart,
CreditsEnd: p.CreditsEnd,
RecapStart: p.RecapStart,
RecapEnd: p.RecapEnd,
PreviewStart: p.PreviewStart,
PreviewEnd: p.PreviewEnd,
MarkersSource: p.Source,
MarkersProvider: p.Provider,
MarkersConfidence: p.Confidence,
MarkersAlgorithm: p.Algorithm,
IntroStart: p.Intro.Start,
IntroEnd: p.Intro.End,
CreditsStart: p.Credits.Start,
CreditsEnd: p.Credits.End,
RecapStart: p.Recap.Start,
RecapEnd: p.Recap.End,
PreviewStart: p.Preview.Start,
PreviewEnd: p.Preview.End,
MarkersSource: p.SummarySource(),
MarkersConfidence: p.SummaryConfidence(),
IntroProvenance: segmentProvenance(p.Intro),
CreditsProvenance: segmentProvenance(p.Credits),
RecapProvenance: segmentProvenance(p.Recap),
PreviewProvenance: segmentProvenance(p.Preview),
}
}
func segmentProvenance(s markers.SegmentPayload) *scanner.SegmentProvenance {
if !s.Present() {
return nil
}
return &scanner.SegmentProvenance{
Source: s.Source,
Provider: s.Provider,
Confidence: s.Confidence,
Algorithm: s.Algorithm,
}
}
internal/api/handlers/playback_sessions.go
+7 -3
View File
@@ -42,6 +42,7 @@ type playbackSessionRow struct {
FileDuration *int `json:"file_duration"`
StartedAt time.Time `json:"started_at"`
UpdatedAt time.Time `json:"updated_at"`
PositionSeconds float64 `json:"position_seconds"`
IsPaused bool `json:"is_paused"`
HasPlaybackControl bool `json:"has_playback_control"`
ClientIP string `json:"client_ip,omitempty"`
@@ -53,6 +54,7 @@ type playbackSessionRow struct {
TargetVideoCodec string `json:"target_video_codec,omitempty"`
TargetAudioCodec string `json:"target_audio_codec,omitempty"`
TargetBitrateKbps *int `json:"target_bitrate_kbps"`
TranscodeHWAccel string `json:"transcode_hw_accel,omitempty"`
SourceContainer string `json:"source_container,omitempty"`
SourceBitrateKbps *int `json:"source_bitrate_kbps"`
SourceVideoCodec string `json:"source_video_codec,omitempty"`
@@ -143,6 +145,7 @@ func (l *PlaybackSessionsLoader) Load(
mf.duration,
s.started_at,
s.updated_at,
COALESCE(s.position_seconds, 0),
COALESCE(s.is_paused, FALSE),
COALESCE(s.has_websocket, FALSE),
COALESCE(HOST(s.client_ip), ''),
@@ -154,6 +157,7 @@ func (l *PlaybackSessionsLoader) Load(
COALESCE(s.target_video_codec, ''),
COALESCE(s.target_audio_codec, ''),
s.target_bitrate_kbps,
COALESCE(s.transcode_hw_accel, ''),
COALESCE(mf.container, ''),
mf.bitrate,
COALESCE(mf.codec_video, ''),
@@ -199,9 +203,9 @@ func (l *PlaybackSessionsLoader) Load(
&s.MediaTitle, &s.MediaType, &s.SeriesName, &s.EpisodeName, &s.SeasonNumber, &s.EpisodeNumber,
&posterPath,
&s.PlayMethod, &s.ReportingNode, &s.NodeDisplayName, &s.FileDuration, &s.StartedAt, &s.UpdatedAt,
&s.IsPaused, &s.HasPlaybackControl, &s.ClientIP, &s.AudioTrackIndex, &s.TranscodeAudio, &streamBitrateKbps,
&s.PositionSeconds, &s.IsPaused, &s.HasPlaybackControl, &s.ClientIP, &s.AudioTrackIndex, &s.TranscodeAudio, &streamBitrateKbps,
&s.TranscodeNodeURL, &s.TargetResolution, &s.TargetVideoCodec, &s.TargetAudioCodec, &targetBitrateKbps,
&s.SourceContainer, &sourceBitrateKbps, &s.SourceVideoCodec, &s.SourceVideoResolution,
&s.TranscodeHWAccel, &s.SourceContainer, &sourceBitrateKbps, &s.SourceVideoCodec, &s.SourceVideoResolution,
&s.SourceAudioCodec, &sourceAudioChannels, &audioTracksJSON, &s.RequestedVideoCodec, &s.RequestedVideoResolution,
); err != nil {
return nil, fmt.Errorf("scanning playback session: %w", err)
@@ -314,7 +318,7 @@ func sessionComponentDecision(playMethod string, transcodeAudio bool, targetVide
}
audioDec := "transcode"
if !transcodeAudio {
audioDec = "direct"
audioDec = "remux"
}
return videoDec, audioDec
default:
internal/api/handlers/playback_sessions_test.go
+14
View File
@@ -0,0 +1,14 @@
package handlers
import "testing"
func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T) {
videoDecision, audioDecision := sessionComponentDecision("transcode", false, "copy")
if videoDecision != "remux" {
t.Fatalf("videoDecision = %q, want remux", videoDecision)
}
if audioDecision != "remux" {
t.Fatalf("audioDecision = %q, want remux", audioDecision)
}
}
internal/api/handlers/plugins.go
+181 -16
View File
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
@@ -20,9 +21,14 @@ import (
"github.com/Silo-Server/silo-server/internal/metadata"
"github.com/Silo-Server/silo-server/internal/pluginhost"
"github.com/Silo-Server/silo-server/internal/plugins"
"github.com/Silo-Server/silo-server/internal/uploads"
)
const maxPluginUploadSize = 256 << 20
const (
maxPluginUploadSize = 256 << 20
maxPluginUploadChunkSize = 1 << 20
defaultPluginChunkSize = 512 << 10
)
type PluginHandler struct {
repositories *plugins.RepositoryStore
@@ -33,6 +39,7 @@ type PluginHandler struct {
proxy *plugins.HTTPProxy
chainRepo *metadata.ChainRepository
imageResolver *metadata.PluginImageResolver
uploads *uploads.Manager
}
func NewPluginHandler(
@@ -54,6 +61,10 @@ func NewPluginHandler(
proxy: proxy,
chainRepo: chainRepo,
imageResolver: imageResolver,
uploads: uploads.NewManager(uploads.ManagerOptions{
MaxSize: maxPluginUploadSize,
MaxChunkSize: maxPluginUploadChunkSize,
}),
}
}
@@ -97,6 +108,12 @@ type userPluginSettingsRequest struct {
Values map[string]string `json:"values"`
}
type pluginChunkedUploadCreateRequest struct {
Filename string `json:"filename"`
SizeBytes int64 `json:"size_bytes"`
ChunkSize int64 `json:"chunk_size,omitempty"`
}
type pluginRepositoryResponse struct {
ID int `json:"id"`
URL string `json:"url"`
@@ -248,6 +265,18 @@ type pluginTaskBindingUpdateResponse struct {
RestartRequired bool `json:"restart_required"`
}
type pluginChunkedUploadSessionResponse struct {
UploadID string `json:"upload_id"`
Filename string `json:"filename"`
SizeBytes int64 `json:"size_bytes"`
ChunkSize int64 `json:"chunk_size"`
TotalChunks int `json:"total_chunks"`
ReceivedChunks int `json:"received_chunks"`
ReceivedBytes int64 `json:"received_bytes"`
Complete bool `json:"complete"`
ExpiresAt time.Time `json:"expires_at"`
}
func (h *PluginHandler) HandleListRepositories(w http.ResponseWriter, r *http.Request) {
repositories, err := h.repositories.List(r.Context())
if err != nil {
@@ -491,27 +520,114 @@ func (h *PluginHandler) HandleUploadInstallation(w http.ResponseWriter, r *http.
return
}
uploadData, err := os.ReadFile(tempPath)
if err != nil {
slog.Error("reading temp plugin upload file", "filename", header.Filename, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to process plugin upload")
return
}
var result *plugins.InstallResult
if isZipUpload(uploadData) {
result, err = h.service.InstallLocal(r.Context(), plugins.InstallArchiveRequest{
ArchivePath: tempPath,
})
} else {
result, err = h.service.InstallBinaryUpload(r.Context(), uploadData)
}
result, err := h.installUploadedPlugin(r.Context(), tempPath)
if err != nil {
slog.Error("installing uploaded plugin", "filename", header.Filename, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to install uploaded plugin")
return
}
h.writeUploadedPluginResponse(w, r, result)
}
func (h *PluginHandler) HandleCreateChunkedUpload(w http.ResponseWriter, r *http.Request) {
var req pluginChunkedUploadCreateRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
if req.ChunkSize == 0 {
req.ChunkSize = defaultPluginChunkSize
}
session, err := h.uploads.Create(uploads.CreateRequest{
Filename: req.Filename,
SizeBytes: req.SizeBytes,
ChunkSize: req.ChunkSize,
})
if err != nil {
status, message := uploadErrorResponse(err)
writeError(w, status, "upload_error", message)
return
}
writeJSON(w, http.StatusCreated, toPluginChunkedUploadSessionResponse(session))
}
func (h *PluginHandler) HandleUploadChunk(w http.ResponseWriter, r *http.Request) {
uploadID := chi.URLParam(r, "upload_id")
chunkIndex, err := strconv.Atoi(chi.URLParam(r, "chunk_index"))
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid chunk index")
return
}
r.Body = http.MaxBytesReader(w, r.Body, h.uploads.MaxChunkSize()+1)
defer r.Body.Close()
session, err := h.uploads.PutChunk(r.Context(), uploadID, chunkIndex, r.Body, r.ContentLength)
if err != nil {
status, message := uploadErrorResponse(err)
writeError(w, status, "upload_error", message)
return
}
writeJSON(w, http.StatusOK, toPluginChunkedUploadSessionResponse(session))
}
func (h *PluginHandler) HandleCompleteChunkedUpload(w http.ResponseWriter, r *http.Request) {
uploadID := chi.URLParam(r, "upload_id")
upload, err := h.uploads.Complete(uploadID)
if err != nil {
status, message := uploadErrorResponse(err)
writeError(w, status, "upload_error", message)
return
}
defer upload.Cleanup()
result, err := h.installUploadedPlugin(r.Context(), upload.Path)
if err != nil {
slog.Error("installing chunked plugin upload",
"filename", upload.Filename,
"upload_id", upload.ID,
"error", err,
)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to install uploaded plugin")
return
}
h.writeUploadedPluginResponse(w, r, result)
}
func (h *PluginHandler) HandleCancelChunkedUpload(w http.ResponseWriter, r *http.Request) {
if err := h.uploads.Cancel(chi.URLParam(r, "upload_id")); err != nil && !errors.Is(err, uploads.ErrNotFound) {
status, message := uploadErrorResponse(err)
writeError(w, status, "upload_error", message)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *PluginHandler) installUploadedPlugin(ctx context.Context, path string) (*plugins.InstallResult, error) {
zipUpload, err := isZipUploadFile(path)
if err != nil {
return nil, err
}
if zipUpload {
return h.service.InstallLocal(ctx, plugins.InstallArchiveRequest{
ArchivePath: path,
})
}
uploadData, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read uploaded plugin binary: %w", err)
}
return h.service.InstallBinaryUpload(ctx, uploadData)
}
func (h *PluginHandler) writeUploadedPluginResponse(w http.ResponseWriter, r *http.Request, result *plugins.InstallResult) {
h.syncMetadataProviders(r.Context(), result.Installation)
h.syncImageResolvers(r.Context(), result.Installation)
@@ -593,6 +709,55 @@ func isZipUpload(data []byte) bool {
bytes.Equal(data[:4], []byte("PK\x07\x08"))
}
func isZipUploadFile(path string) (bool, error) {
file, err := os.Open(path)
if err != nil {
return false, fmt.Errorf("open uploaded plugin file: %w", err)
}
defer file.Close()
var header [4]byte
n, err := io.ReadFull(file, header[:])
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
return false, fmt.Errorf("read uploaded plugin header: %w", err)
}
return isZipUpload(header[:n]), nil
}
func toPluginChunkedUploadSessionResponse(session uploads.SessionInfo) pluginChunkedUploadSessionResponse {
return pluginChunkedUploadSessionResponse{
UploadID: session.ID,
Filename: session.Filename,
SizeBytes: session.SizeBytes,
ChunkSize: session.ChunkSize,
TotalChunks: session.TotalChunks,
ReceivedChunks: session.ReceivedChunks,
ReceivedBytes: session.ReceivedBytes,
Complete: session.Complete,
ExpiresAt: session.ExpiresAt,
}
}
func uploadErrorResponse(err error) (int, string) {
var maxBytesErr *http.MaxBytesError
switch {
case errors.As(err, &maxBytesErr), errors.Is(err, uploads.ErrTooLarge):
return http.StatusRequestEntityTooLarge, "Upload exceeds the maximum allowed size"
case errors.Is(err, uploads.ErrNotFound):
return http.StatusNotFound, "Upload session not found"
case errors.Is(err, uploads.ErrExpired):
return http.StatusGone, "Upload session expired"
case errors.Is(err, uploads.ErrIncomplete):
return http.StatusConflict, "Upload session is incomplete"
case errors.Is(err, uploads.ErrAlreadyCompleted):
return http.StatusConflict, "Upload session is already complete"
case errors.Is(err, uploads.ErrInvalidChunk), errors.Is(err, uploads.ErrInvalidRequest):
return http.StatusBadRequest, err.Error()
default:
return http.StatusInternalServerError, "Failed to process upload"
}
}
func (h *PluginHandler) HandleUpdateInstallation(w http.ResponseWriter, r *http.Request) {
id, err := parseNamedIDParam(r, "id")
if err != nil {
internal/api/handlers/progress.go
+1 -1
View File
@@ -258,7 +258,7 @@ func (h *ProgressHandler) HandleSyncProgress(w http.ResponseWriter, r *http.Requ
item.MediaItemID,
"",
"progress",
"progress.updated",
userStateEventState{},
)
}
}
internal/api/handlers/request_proxy.go
+29
View File
@@ -0,0 +1,29 @@
package handlers
import (
"net/http"
"strings"
)
// forwardedHost returns the public host a reverse proxy or CDN forwarded the
// request on behalf of, taken from the first X-Forwarded-Host value, or "" when
// the header is absent.
//
// Behind a TLS-terminating CDN/proxy, r.Host is the internal origin host the
// proxy dialed, while the public host the browser actually used arrives here.
// Callers that need to reason about the client-facing host (origin checks,
// absolute URL construction) must prefer this over r.Host.
func forwardedHost(r *http.Request) string {
forwarded := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
if forwarded == "" {
return ""
}
// Multiple proxy hops produce a comma-separated list; the first entry is
// the original client-facing host.
if comma := strings.IndexByte(forwarded, ','); comma >= 0 {
forwarded = forwarded[:comma]
}
return strings.TrimSpace(forwarded)
}
internal/api/handlers/requests.go
+100 -38
View File
@@ -35,8 +35,9 @@ type RequestService interface {
GetUserLimit(ctx context.Context, viewer mediarequests.Viewer, userID int) (*mediarequests.UserLimit, error)
UpsertUserLimit(ctx context.Context, viewer mediarequests.Viewer, limit mediarequests.UserLimit) (*mediarequests.UserLimit, error)
ListIntegrations(ctx context.Context, viewer mediarequests.Viewer) ([]mediarequests.Integration, error)
UpsertIntegration(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error)
UpsertIntegrations(ctx context.Context, viewer mediarequests.Viewer, integrations []mediarequests.Integration) ([]mediarequests.Integration, error)
CreateIntegration(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error)
UpdateIntegration(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error)
DeleteIntegration(ctx context.Context, viewer mediarequests.Viewer, id string) error
LoadIntegrationOptions(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.IntegrationOptions, error)
ListStudios(ctx context.Context, viewer mediarequests.Viewer) ([]mediarequests.DiscoverBrandCard, error)
@@ -422,26 +423,53 @@ func (h *RequestsHandler) HandleListIntegrations(w http.ResponseWriter, r *http.
}{Integrations: toIntegrationResponses(integrations)})
}
func (h *RequestsHandler) HandleUpdateIntegrations(w http.ResponseWriter, r *http.Request) {
func (h *RequestsHandler) HandleCreateIntegration(w http.ResponseWriter, r *http.Request) {
viewer, ok := requestViewer(w, r, false)
if !ok {
return
}
var body struct {
Integrations []mediarequests.Integration `json:"integrations"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
var integration mediarequests.Integration
if err := json.NewDecoder(r.Body).Decode(&integration); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
updated, err := h.service.UpsertIntegrations(r.Context(), viewer, body.Integrations)
created, err := h.service.CreateIntegration(r.Context(), viewer, integration)
if err != nil {
writeRequestServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, struct {
Integrations []requestIntegrationResponse `json:"integrations"`
}{Integrations: toIntegrationResponses(updated)})
writeJSON(w, http.StatusCreated, requestIntegrationResponseFrom(*created))
}
func (h *RequestsHandler) HandleUpdateIntegration(w http.ResponseWriter, r *http.Request) {
viewer, ok := requestViewer(w, r, false)
if !ok {
return
}
var integration mediarequests.Integration
if err := json.NewDecoder(r.Body).Decode(&integration); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
integration.ID = chi.URLParam(r, "id")
updated, err := h.service.UpdateIntegration(r.Context(), viewer, integration)
if err != nil {
writeRequestServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, requestIntegrationResponseFrom(*updated))
}
func (h *RequestsHandler) HandleDeleteIntegration(w http.ResponseWriter, r *http.Request) {
viewer, ok := requestViewer(w, r, false)
if !ok {
return
}
if err := h.service.DeleteIntegration(r.Context(), viewer, chi.URLParam(r, "id")); err != nil {
writeRequestServiceError(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *RequestsHandler) HandleLoadIntegrationOptions(w http.ResponseWriter, r *http.Request) {
@@ -456,7 +484,9 @@ func (h *RequestsHandler) HandleLoadIntegrationOptions(w http.ResponseWriter, r
return
}
}
integration.Kind = chi.URLParam(r, "kind")
if id := strings.TrimSpace(chi.URLParam(r, "id")); id != "" {
integration.ID = id
}
options, err := h.service.LoadIntegrationOptions(r.Context(), viewer, integration)
if err != nil {
writeRequestServiceError(w, err)
@@ -558,37 +588,69 @@ func parsePositivePathInt(w http.ResponseWriter, r *http.Request, key string) (i
}
type requestIntegrationResponse struct {
Kind string `json:"kind"`
Enabled bool `json:"enabled"`
BaseURL string `json:"base_url"`
HasAPIKey bool `json:"has_api_key"`
RootFolder string `json:"root_folder"`
QualityProfileID *int `json:"quality_profile_id,omitempty"`
Tags []int `json:"tags"`
Options map[string]any `json:"options"`
LastCheckAt *time.Time `json:"last_check_at,omitempty"`
LastCheckStatus string `json:"last_check_status,omitempty"`
LastCheckError string `json:"last_check_error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
ID string `json:"id"`
Name string `json:"name"`
Kind string `json:"kind"`
Enabled bool `json:"enabled"`
Is4K bool `json:"is_4k"`
IsDefault bool `json:"is_default"`
IsDefault4K bool `json:"is_default_4k"`
AnimeEnabled bool `json:"anime_enabled"`
AnimeQualityProfileID *int `json:"anime_quality_profile_id,omitempty"`
AnimeRootFolder string `json:"anime_root_folder,omitempty"`
AnimeTags []int `json:"anime_tags"`
BaseURL string `json:"base_url"`
HasAPIKey bool `json:"has_api_key"`
RootFolder string `json:"root_folder"`
QualityProfileID *int `json:"quality_profile_id,omitempty"`
Tags []int `json:"tags"`
Options map[string]any `json:"options"`
LastCheckAt *time.Time `json:"last_check_at,omitempty"`
LastCheckStatus string `json:"last_check_status,omitempty"`
LastCheckError string `json:"last_check_error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
func requestIntegrationResponseFrom(integration mediarequests.Integration) requestIntegrationResponse {
// Normalize nil slices so they serialize as [] (not null); the frontend
// types them as number[] and indexes/maps over them.
tags := integration.Tags
if tags == nil {
tags = []int{}
}
animeTags := integration.AnimeTags
if animeTags == nil {
animeTags = []int{}
}
return requestIntegrationResponse{
ID: integration.ID,
Name: integration.Name,
Kind: integration.Kind,
Enabled: integration.Enabled,
Is4K: integration.Is4K,
IsDefault: integration.IsDefault,
IsDefault4K: integration.IsDefault4K,
AnimeEnabled: integration.AnimeEnabled,
AnimeQualityProfileID: integration.AnimeQualityProfileID,
AnimeRootFolder: integration.AnimeRootFolder,
AnimeTags: animeTags,
BaseURL: integration.BaseURL,
HasAPIKey: strings.TrimSpace(integration.APIKeyRef) != "",
RootFolder: integration.RootFolder,
QualityProfileID: integration.QualityProfileID,
Tags: tags,
Options: integration.Options,
LastCheckAt: integration.LastCheckAt,
LastCheckStatus: integration.LastCheckStatus,
LastCheckError: integration.LastCheckError,
UpdatedAt: integration.UpdatedAt,
}
}
func toIntegrationResponses(integrations []mediarequests.Integration) []requestIntegrationResponse {
out := make([]requestIntegrationResponse, 0, len(integrations))
for _, integration := range integrations {
out = append(out, requestIntegrationResponse{
Kind: integration.Kind,
Enabled: integration.Enabled,
BaseURL: integration.BaseURL,
HasAPIKey: strings.TrimSpace(integration.APIKeyRef) != "",
RootFolder: integration.RootFolder,
QualityProfileID: integration.QualityProfileID,
Tags: integration.Tags,
Options: integration.Options,
LastCheckAt: integration.LastCheckAt,
LastCheckStatus: integration.LastCheckStatus,
LastCheckError: integration.LastCheckError,
UpdatedAt: integration.UpdatedAt,
})
out = append(out, requestIntegrationResponseFrom(integration))
}
return out
}
internal/api/handlers/requests_test.go
+8 -4
View File
@@ -127,12 +127,16 @@ func (f *fakeRequestService) ListIntegrations(context.Context, mediarequests.Vie
return nil, nil
}
func (f *fakeRequestService) UpsertIntegration(context.Context, mediarequests.Viewer, mediarequests.Integration) (*mediarequests.Integration, error) {
return nil, nil
func (f *fakeRequestService) CreateIntegration(_ context.Context, _ mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error) {
return &integration, nil
}
func (f *fakeRequestService) UpsertIntegrations(context.Context, mediarequests.Viewer, []mediarequests.Integration) ([]mediarequests.Integration, error) {
return nil, nil
func (f *fakeRequestService) UpdateIntegration(_ context.Context, _ mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error) {
return &integration, nil
}
func (f *fakeRequestService) DeleteIntegration(context.Context, mediarequests.Viewer, string) error {
return nil
}
func (f *fakeRequestService) LoadIntegrationOptions(context.Context, mediarequests.Viewer, mediarequests.Integration) (*mediarequests.IntegrationOptions, error) {

Some files were not shown because too many files have changed in this diff Show More