Compare commits
222
Commits
@@ -1,12 +1,6 @@
|
||||
# Silo Docker deployment
|
||||
# For the default docker compose setup, MEDIA_ROOT is the main value you need to set.
|
||||
|
||||
# Optional: layer compose files through env. Leave unset for CPU-only defaults.
|
||||
# Linux/macOS example for enabling NVIDIA override:
|
||||
# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
|
||||
# Windows example:
|
||||
# COMPOSE_FILE=docker-compose.yml;docker-compose.nvidia.yml
|
||||
|
||||
# Required for docker compose: host path to your media library.
|
||||
# Example: MEDIA_ROOT=/srv/media
|
||||
MEDIA_ROOT=/srv/media
|
||||
@@ -46,19 +40,6 @@ POSTGRES_SHM_SIZE=8gb
|
||||
# PROXY_PORT=8083
|
||||
# TRANSCODE_PORT=8082
|
||||
|
||||
# Optional NVIDIA override controls (used by docker-compose.nvidia.yml).
|
||||
# NVIDIA_GPU_COUNT=1
|
||||
|
||||
# REQUIRED: master key for at-rest credential encryption. Silo refuses to start
|
||||
# without it. Integration API keys, S3 keys, and other server-owned secrets are
|
||||
# encrypted under a key derived from this value, so it must be at least 32
|
||||
# characters and kept secret. Generate one with:
|
||||
# openssl rand -base64 48
|
||||
# Back it up SEPARATELY from your database dumps (treat it like a CA private
|
||||
# key): losing it makes every encrypted secret unrecoverable. See
|
||||
# docs/architecture/secret-encryption.md.
|
||||
# SECRET_KEY=replace-with-output-of-openssl-rand-base64-48
|
||||
|
||||
# Run from source / advanced overrides
|
||||
# Only DATABASE_URL is required when running Silo outside the default docker compose stack.
|
||||
# DATABASE_URL=postgres://silo:password@localhost:5432/silo
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Repository Guidelines
|
||||
|
||||
## Project Structure & Module Organization
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/sql/` as Goose SQL migrations. Legacy converted migrations intentionally keep their original numeric versions so existing `schema_versions` rows can bootstrap cleanly into Goose. New migrations must use timestamped filenames created with `make migrate-create NAME=add_thing`; do not run `goose fix` or create paired `.up.sql` / `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/` as paired numbered `.up.sql` and `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
|
||||
When creating or editing `docs/superpowers/specs/` or `docs/superpowers/plans/`, never include local absolute filesystem paths or transient worktree IDs. Use repository-relative paths and wording like "Commands assume the repository root is the cwd."
|
||||
|
||||
@@ -53,7 +53,6 @@ Use the checked-in `Makefile` for the common paths:
|
||||
- `make dev-frontend`: start the Vite dev server with HMR
|
||||
- `make dev-proxy` / `make dev-transcode`: run standalone worker modes
|
||||
- `make lint`: run `golangci-lint` and frontend ESLint
|
||||
- `make migrate-status` / `make migrate-up`: inspect or apply Goose migrations through Silo's legacy-safe bootstrapping runner
|
||||
|
||||
Run before opening a merge request:
|
||||
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks migrate-create migrate-validate migrate-status migrate-up
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks
|
||||
|
||||
GIT_COMMON_DIR := $(strip $(shell git rev-parse --git-common-dir 2>/dev/null))
|
||||
MAIN_CHECKOUT_ROOT := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/..))
|
||||
SHARED_MAKEFILE_LOCAL := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/../Makefile.local))
|
||||
DEFAULT_PLUGIN_SDK_DIR := $(abspath ../silo-plugin-sdk)
|
||||
SHARED_PLUGIN_SDK_DIR := $(if $(MAIN_CHECKOUT_ROOT),$(abspath $(MAIN_CHECKOUT_ROOT)/../silo-plugin-sdk))
|
||||
GOOSE := go run github.com/pressly/goose/v3/cmd/goose@v3.27.1
|
||||
GOOSE_DIR := migrations/sql
|
||||
ENV_FILE ?= .env
|
||||
|
||||
ifneq ($(wildcard $(DEFAULT_PLUGIN_SDK_DIR)),)
|
||||
DEV_PLUGIN_SDK_DIR ?= $(DEFAULT_PLUGIN_SDK_DIR)
|
||||
@@ -57,23 +54,6 @@ lint:
|
||||
verify-local-paths:
|
||||
scripts/check-local-path-leaks.sh
|
||||
|
||||
# Create a timestamped Goose SQL migration. Usage: make migrate-create NAME=add_thing
|
||||
migrate-create:
|
||||
@if [ -z "$(NAME)" ]; then echo "usage: make migrate-create NAME=add_thing"; exit 1; fi
|
||||
$(GOOSE) -dir $(GOOSE_DIR) create "$(NAME)" sql
|
||||
|
||||
# Validate Goose migration annotations and SQL parsing without touching a database.
|
||||
migrate-validate:
|
||||
$(GOOSE) -dir $(GOOSE_DIR) validate
|
||||
|
||||
# Show Goose migration status through Silo's bootstrapping runner.
|
||||
migrate-status:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-status
|
||||
|
||||
# Apply pending Goose migrations through Silo's bootstrapping runner.
|
||||
migrate-up:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-only
|
||||
|
||||
# Install repo-local git hooks for this checkout/worktree.
|
||||
install-hooks:
|
||||
@existing="$$(git config --local core.hooksPath 2>/dev/null || true)"; \
|
||||
|
||||
@@ -34,29 +34,6 @@ The easiest way to run Silo is with Docker Compose. The default stack assumes yo
|
||||
|
||||
If you already have PostgreSQL and Redis available, omit those bundled service examples from compose and point Silo at your existing `DATABASE_URL` and `REDIS_URL` instead.
|
||||
|
||||
### Optional NVIDIA/NVENC
|
||||
|
||||
GPU support is kept out of the default compose file so hosts without NVIDIA drivers work unchanged.
|
||||
|
||||
Install the NVIDIA Container Toolkit and use a Docker Compose version with GPU reservation support before enabling this override.
|
||||
|
||||
Use the optional override file when you want NVENC:
|
||||
|
||||
```sh
|
||||
docker compose -f docker-compose.yml -f docker-compose.nvidia.yml up -d
|
||||
```
|
||||
|
||||
If you want this controlled from `.env`, set `COMPOSE_FILE`:
|
||||
|
||||
```dotenv
|
||||
COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
|
||||
NVIDIA_GPU_COUNT=1
|
||||
```
|
||||
|
||||
Windows uses `;` instead of `:` between compose files.
|
||||
|
||||
Then `docker compose up -d` will include the NVIDIA override automatically.
|
||||
|
||||
4. **Configure through the admin UI**
|
||||
|
||||
Add libraries, users, metadata providers, and playback settings from the web interface.
|
||||
@@ -201,36 +178,8 @@ Technical notes:
|
||||
| `make dev-backend` | Run Go backend (integrated mode) |
|
||||
| `make dev-proxy` | Run a standalone proxy node |
|
||||
| `make dev-transcode` | Run a standalone transcode node |
|
||||
| `make migrate-create NAME=add_thing` | Create a timestamped Goose SQL migration |
|
||||
| `make migrate-validate` | Validate Goose migration files without touching a database |
|
||||
| `make migrate-status` | Show Goose migration status using Silo's bootstrapping runner |
|
||||
| `make migrate-up` | Apply pending Goose migrations using Silo's bootstrapping runner |
|
||||
| `make clean` | Remove build artifacts |
|
||||
|
||||
### Database Migrations
|
||||
|
||||
PostgreSQL schema migrations are managed by Goose. Migration SQL files live in
|
||||
`migrations/sql/` and use Goose annotations. Converted legacy migrations keep
|
||||
their original numeric versions so existing `schema_versions` rows can bootstrap
|
||||
cleanly into Goose without replaying old SQL. New migrations should be created
|
||||
with timestamped filenames:
|
||||
|
||||
```sh
|
||||
make migrate-create NAME=add_thing
|
||||
make migrate-validate
|
||||
```
|
||||
|
||||
Do not run `goose fix`; timestamped migrations are the repository policy because
|
||||
they avoid version collisions across parallel PRs. The existing `001`-style
|
||||
files are historical compatibility records, not the naming pattern for new work.
|
||||
Runtime migrations are applied by the integrated/API server only. Proxy and
|
||||
transcode modes never mutate schema.
|
||||
For existing installs, use `make migrate-status` and `make migrate-up` rather
|
||||
than invoking the Goose CLI directly; those targets copy legacy
|
||||
`schema_versions` rows into `public.goose_db_version` under the migration lock
|
||||
before reading or applying migrations. Set `ENV_FILE=path/to/.env` when the
|
||||
database URL should be read from a non-default env file.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```sh
|
||||
@@ -336,5 +285,5 @@ internal/
|
||||
scanner/ Media file discovery and FFProbe
|
||||
worker/ Background jobs (scan, match, reconcile)
|
||||
web/ React + TypeScript frontend (Vite, Tailwind, shadcn/ui)
|
||||
migrations/sql/ Goose-managed PostgreSQL schema migrations
|
||||
migrations/ PostgreSQL schema migrations
|
||||
```
|
||||
|
||||
+93
-380
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -14,10 +16,8 @@ import (
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -25,11 +25,11 @@ import (
|
||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
sdkcapability "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/activitylog"
|
||||
@@ -56,6 +56,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/logfilter"
|
||||
"github.com/Silo-Server/silo-server/internal/logstream"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/markers/introdb"
|
||||
"github.com/Silo-Server/silo-server/internal/mdblist"
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
@@ -77,7 +78,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/s3client"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/Silo-Server/silo-server/internal/scanqueue"
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
"github.com/Silo-Server/silo-server/internal/server"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
@@ -142,7 +142,7 @@ func mustGetSetting(store interface {
|
||||
func configureOperationalLogging(
|
||||
ctx context.Context,
|
||||
pool *pgxpool.Pool,
|
||||
settingsRepo catalog.SettingsStore,
|
||||
settingsRepo *catalog.ServerSettingsRepo,
|
||||
redisCfg config.RedisConfig,
|
||||
logStreamHub *logstream.Hub,
|
||||
baseHandler slog.Handler,
|
||||
@@ -253,42 +253,8 @@ func maybeApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, appMaxCon
|
||||
}
|
||||
}
|
||||
|
||||
// runCredentialBackfills sweeps any plaintext server-owned credential to
|
||||
// ciphertext on the primary (migration-running) node. All passes are
|
||||
// best-effort: a failed row leaves the prior plaintext (no new exposure) and
|
||||
// still reads via the read-path pass-through, so a backfill error must never
|
||||
// block boot. The sensitive-settings pass runs first so the arr
|
||||
// resolve-then-encrypt pass sees consistent referenced settings.
|
||||
func runCredentialBackfills(ctx context.Context, pool *pgxpool.Pool, cipher *secret.Cipher, settings *catalog.EncryptedSettingsRepo) {
|
||||
settingsN, err := settings.BackfillSensitiveSettings(ctx)
|
||||
if err != nil {
|
||||
slog.Error("secret backfill: sensitive settings", "error", err)
|
||||
}
|
||||
columnsN, err := secret.BackfillColumns(ctx, pool, cipher, secret.ColumnBackfillTargets())
|
||||
if err != nil {
|
||||
slog.Error("secret backfill: credential columns", "error", err)
|
||||
}
|
||||
historyServersN, err := historyimport.NewRepository(pool, cipher).BackfillSessionServerSecrets(ctx)
|
||||
if err != nil {
|
||||
slog.Error("secret backfill: history import session server credentials", "error", err)
|
||||
}
|
||||
// The arr resolver is the encrypting settings decorator: it decrypts a
|
||||
// sensitive target (e.g. requests.radarr.api_key) or passes through a
|
||||
// plaintext custom key, exactly replicating the deleted resolveAPIKey.
|
||||
arrN, err := secret.BackfillReferencedColumns(ctx, pool, cipher, settings.Get, secret.ArrKeyBackfillTargets())
|
||||
if err != nil {
|
||||
slog.Error("secret backfill: arr api keys", "error", err)
|
||||
}
|
||||
if total := settingsN + columnsN + historyServersN + arrN; total > 0 {
|
||||
slog.Info("secret backfill: encrypted plaintext credentials at rest",
|
||||
"settings", settingsN, "columns", columnsN, "history_session_servers", historyServersN, "arr_keys", arrN, "total", total)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
envFile := flag.String("env", ".env", "path to .env bootstrap file")
|
||||
migrateOnly := flag.Bool("migrate-only", false, "apply database migrations and exit")
|
||||
migrateStatus := flag.Bool("migrate-status", false, "show database migration status and exit")
|
||||
flag.Parse()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -299,15 +265,6 @@ func main() {
|
||||
log.Fatalf("bootstrap: %v", err)
|
||||
}
|
||||
|
||||
// Construct the at-rest credential cipher from SECRET_KEY immediately after
|
||||
// bootstrap, before any settings repo is built. It is threaded explicitly as
|
||||
// a dependency into every repo that stores a server-owned secret — never a
|
||||
// package-level global.
|
||||
dataCipher, err := secret.New(bc.SecretKey)
|
||||
if err != nil {
|
||||
log.Fatalf("secret cipher: %v", err)
|
||||
}
|
||||
|
||||
// Step 2: Connect to PostgreSQL (bootstrap pool with default max connections)
|
||||
bootstrapDBCfg := config.DatabaseConfig{URL: bc.DatabaseURL, MaxConnections: 20}
|
||||
pool, err := database.NewPool(ctx, bootstrapDBCfg)
|
||||
@@ -317,52 +274,12 @@ func main() {
|
||||
defer pool.Close()
|
||||
slog.Info("connected to PostgreSQL")
|
||||
|
||||
if *migrateStatus {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
statuses, statusErr := database.MigrationStatuses(migCtx, pool, migrations.FS, "sql")
|
||||
migCancel()
|
||||
if statusErr != nil {
|
||||
log.Fatalf("failed to read migration status: %v", statusErr)
|
||||
}
|
||||
|
||||
fmt.Printf("%-8s %8s %-25s %s\n", "STATE", "VERSION", "APPLIED_AT", "MIGRATION")
|
||||
for _, status := range statuses {
|
||||
appliedAt := "-"
|
||||
if !status.AppliedAt.IsZero() {
|
||||
appliedAt = status.AppliedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
source := status.Source
|
||||
if source != "" {
|
||||
source = filepath.Base(source)
|
||||
} else {
|
||||
source = "-"
|
||||
}
|
||||
fmt.Printf("%-8s %8d %-25s %s\n", status.State, status.Version, appliedAt, source)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if *migrateOnly {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql")
|
||||
migCancel()
|
||||
if migErr != nil {
|
||||
log.Fatalf("failed to run migrations: %v", migErr)
|
||||
}
|
||||
slog.Info("database migrations applied")
|
||||
return
|
||||
}
|
||||
|
||||
// Run migrations only for integrated/api modes. Proxy and transcode nodes
|
||||
// should never alter the schema — they may scale independently and would
|
||||
// race or apply migrations before the primary node is deliberately upgraded.
|
||||
// The same gate decides whether this node runs the credential-encryption
|
||||
// backfills: only the primary (migration-running) node sweeps plaintext to
|
||||
// ciphertext; secondary nodes read whatever the primary encrypted.
|
||||
isPrimaryNode := bc.Mode == "integrated" || bc.Mode == "api" || bc.Mode == ""
|
||||
if isPrimaryNode {
|
||||
if bc.Mode == "integrated" || bc.Mode == "api" || bc.Mode == "" {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql"); migErr != nil {
|
||||
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "."); migErr != nil {
|
||||
migCancel()
|
||||
log.Fatalf("failed to run migrations: %v", migErr)
|
||||
}
|
||||
@@ -370,14 +287,8 @@ func main() {
|
||||
slog.Info("database migrations applied")
|
||||
}
|
||||
|
||||
// Step 3: Load settings from DB. settingsRepo is the encrypting decorator so
|
||||
// every consumer (config.LoadFromDB, admin, ABS, watchers) transparently sees
|
||||
// plaintext while sensitive keys rest as ciphertext. The settings backfill
|
||||
// (run after migrations, before this GetAll) is wired further below.
|
||||
settingsRepo := catalog.NewEncryptedSettingsRepo(catalog.NewServerSettingsRepo(pool), dataCipher)
|
||||
if isPrimaryNode {
|
||||
runCredentialBackfills(ctx, pool, dataCipher, settingsRepo)
|
||||
}
|
||||
// Step 3: Load settings from DB
|
||||
settingsRepo := catalog.NewServerSettingsRepo(pool)
|
||||
settings, err := settingsRepo.GetAll(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("loading settings: %v", err)
|
||||
@@ -448,9 +359,7 @@ func main() {
|
||||
log.Fatalf("recreating pool with configured max_connections: %v", err)
|
||||
}
|
||||
}
|
||||
// Re-wrap with the encrypting decorator so the recreated pool's settings repo
|
||||
// still encrypts/decrypts — no raw settings repo may escape into later wiring.
|
||||
settingsRepo = catalog.NewEncryptedSettingsRepo(catalog.NewServerSettingsRepo(pool), dataCipher)
|
||||
settingsRepo = catalog.NewServerSettingsRepo(pool)
|
||||
nodeID := resolveNodeIdentity()
|
||||
|
||||
// Step 9: Validate
|
||||
@@ -479,8 +388,6 @@ func main() {
|
||||
|
||||
appCtx, appCancel := context.WithCancel(ctx)
|
||||
defer appCancel()
|
||||
restartReqCh := make(chan struct{}, 1)
|
||||
var restartRequested atomic.Bool
|
||||
|
||||
eventBus := cache.NewEventBus(cfg.Redis.URL)
|
||||
logStreamHub := logstream.NewHub(nodeID, eventBus)
|
||||
@@ -514,7 +421,7 @@ func main() {
|
||||
DatabaseURL: cfg.Database.URL,
|
||||
JFListen: cfg.JellyfinCompat.Listen,
|
||||
}
|
||||
watcher := nodeconfig.NewWatcher(pool, dataCipher, eventBus, bootstrap)
|
||||
watcher := nodeconfig.NewWatcher(pool, eventBus, bootstrap)
|
||||
if err := watcher.Start(appCtx); err != nil {
|
||||
slog.Error("config watcher start failed", "error", err)
|
||||
os.Exit(1)
|
||||
@@ -573,7 +480,6 @@ func main() {
|
||||
BootstrapSensitiveValues: bootstrapSensitiveValues,
|
||||
AppContext: appCtx,
|
||||
DB: pool,
|
||||
SecretCipher: dataCipher,
|
||||
EventBus: eventBus,
|
||||
LogStreamHub: logStreamHub,
|
||||
RealtimeHub: realtimeHub,
|
||||
@@ -582,20 +488,14 @@ func main() {
|
||||
OpsLogRepo: opsRepo,
|
||||
FFmpegLogSink: playback.NewSlogFFmpegLogSink(slog.Default(), nodeID),
|
||||
PublicURL: os.Getenv("SILO_PUBLIC_URL"),
|
||||
RequestServerRestart: func(context.Context) error {
|
||||
if !restartRequested.CompareAndSwap(false, true) {
|
||||
return handlers.ErrServerRestartAlreadyRequested
|
||||
}
|
||||
restartReqCh <- struct{}{}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
audiobooksService := audiobooks.New(&audiobooksSettingsAdapter{repo: settingsRepo})
|
||||
absCompatEnabled, err := audiobooksService.ABSCompatEnabled(appCtx)
|
||||
audiobooksEnabled, err := audiobooksService.Enabled(appCtx)
|
||||
if err != nil {
|
||||
slog.Warn("Audiobookshelf compatibility disabled; failed to read setting", "err", err)
|
||||
absCompatEnabled = false
|
||||
slog.Warn("audiobooks feature disabled; failed to read setting", "err", err)
|
||||
audiobooksEnabled = false
|
||||
}
|
||||
deps.AudiobooksEnabled = audiobooksEnabled
|
||||
adminJobCancelRegistry := adminjob.NewCancelRegistry()
|
||||
deps.AdminJobCancelRegistry = adminJobCancelRegistry
|
||||
if needsWorkers && deps.DB != nil {
|
||||
@@ -608,34 +508,36 @@ func main() {
|
||||
}
|
||||
if deps.DB != nil {
|
||||
markerRegistry := markers.NewRegistry(slog.Default())
|
||||
markerProviderConfig := markers.NewProviderConfigStore(deps.DB)
|
||||
if err := markerProviderConfig.Reload(appCtx); err != nil {
|
||||
slog.Warn("load marker provider config failed; falling back to registration-order fetch",
|
||||
introdbAPIKey, err := settingsRepo.Get(appCtx, "introdb.api_key")
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
slog.Warn("load introdb.api_key from settings failed; provider will run with no key",
|
||||
"error", err)
|
||||
} else {
|
||||
markerRegistry.UseConfigStore(markerProviderConfig)
|
||||
if deps.EventBus != nil {
|
||||
if err := deps.EventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
|
||||
if event.Type != cache.EventMarkerProviderConfigChanged {
|
||||
return
|
||||
}
|
||||
if err := markerProviderConfig.Reload(appCtx); err != nil {
|
||||
slog.Warn("reload marker provider config failed", "provider", event.Payload, "error", err)
|
||||
}
|
||||
}); err != nil {
|
||||
slog.Warn("subscribe marker provider config reload failed", "error", err)
|
||||
}
|
||||
introdbAPIKey = ""
|
||||
}
|
||||
introdbClient := introdb.NewClient(introdbAPIKey)
|
||||
if err := markerRegistry.Register(introdb.NewProvider(introdbClient)); err != nil {
|
||||
log.Fatalf("register introdb marker provider: %v", err)
|
||||
}
|
||||
deps.OnServerSettingUpdated = func(_ context.Context, key, value string) {
|
||||
if key == "introdb.api_key" {
|
||||
introdbClient.SetAPIKey(value)
|
||||
}
|
||||
}
|
||||
deps.MarkerProviderConfig = markerProviderConfig
|
||||
if eventBus != nil {
|
||||
_ = eventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
|
||||
if event.Type != cache.EventSettingsChanged || event.Payload != "introdb.api_key" {
|
||||
return
|
||||
}
|
||||
value, loadErr := settingsRepo.Get(context.Background(), "introdb.api_key")
|
||||
if loadErr != nil {
|
||||
slog.Warn("introdb api key reload failed", "error", loadErr)
|
||||
return
|
||||
}
|
||||
introdbClient.SetAPIKey(value)
|
||||
})
|
||||
}
|
||||
deps.MarkerRegistry = markerRegistry
|
||||
markerResolver := markers.NewDBExternalIDResolver(deps.DB)
|
||||
deps.MarkerResolver = markerResolver
|
||||
markerContributionStore := markers.NewContributionStore(deps.DB)
|
||||
deps.MarkerContributionStore = markerContributionStore
|
||||
deps.MarkerContributionService = markers.NewContributionService(
|
||||
markerRegistry, markerResolver, markerProviderConfig, markerContributionStore, slog.Default(),
|
||||
)
|
||||
deps.MarkerResolver = markers.NewDBExternalIDResolver(deps.DB)
|
||||
}
|
||||
var watchProviderService *watchsync.Service
|
||||
if deps.DB != nil {
|
||||
@@ -650,7 +552,7 @@ func main() {
|
||||
log.Fatalf("register watch provider: %v", err)
|
||||
}
|
||||
watchProviderService = watchsync.NewService(
|
||||
watchsync.NewPostgresRepository(deps.DB, deps.SecretCipher),
|
||||
watchsync.NewPostgresRepository(deps.DB),
|
||||
watchProviderRegistry,
|
||||
)
|
||||
deps.WatchProviderService = watchProviderService
|
||||
@@ -847,22 +749,6 @@ func main() {
|
||||
installer,
|
||||
plugins.NewHostAdapter(pluginHost),
|
||||
)
|
||||
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
|
||||
markerPluginResolver := markers.NewPluginResolverAdapter(pluginService)
|
||||
pluginService.AddLifecycleHook(func(ctx context.Context) {
|
||||
if err := reloadMarkerPluginProviders(
|
||||
ctx,
|
||||
deps.MarkerRegistry,
|
||||
deps.MarkerProviderConfig,
|
||||
installationStore,
|
||||
runtimeConfigStore,
|
||||
settingsRepo,
|
||||
markerPluginResolver,
|
||||
); err != nil {
|
||||
slog.Warn("reload marker plugin providers failed", "error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := pluginService.PreloadEnabled(appCtx); err != nil {
|
||||
log.Fatalf("preload enabled plugins: %v", err)
|
||||
}
|
||||
@@ -1235,7 +1121,7 @@ func main() {
|
||||
deps.UserStoreProvider = userStoreProvider
|
||||
}
|
||||
if watchProviderService != nil {
|
||||
historyRepo := historyimport.NewRepository(deps.DB, deps.SecretCipher)
|
||||
historyRepo := historyimport.NewRepository(deps.DB)
|
||||
historyIdentity := watchstate.NewStableIdentityResolver(itemRepo, episodeRepo, catalog.NewProviderIDRepository(deps.DB))
|
||||
watchProviderService.
|
||||
WithMatcher(historyimport.NewMatcher(historyRepo)).
|
||||
@@ -1498,15 +1384,9 @@ func main() {
|
||||
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil {
|
||||
taskMgr.Register(tasks.NewScanLibrariesTask(deps.FolderRepo, deps.LibraryScanQueue, deps.EventBus))
|
||||
}
|
||||
taskMgr.Register(tasks.NewCleanupOrphanedMediaItemsTask(catalog.NewOrphanedProvisionalCleaner(deps.DB)))
|
||||
if deps.IntroAnalyzer != nil {
|
||||
taskMgr.Register(tasks.NewDetectIntroMarkersTask(deps.IntroAnalyzer, settingsRepo))
|
||||
}
|
||||
if deps.MarkerContributionService != nil && deps.MarkerProviderConfig != nil && deps.MarkerContributionStore != nil && deps.FileRepo != nil {
|
||||
taskMgr.Register(tasks.NewContributeMarkersTask(
|
||||
deps.MarkerContributionService, deps.MarkerProviderConfig, deps.MarkerContributionStore, deps.FileRepo,
|
||||
))
|
||||
}
|
||||
if chapterBackfiller, ok := deps.ChapterThumbnailQueuer.(*chapterthumbs.Service); ok {
|
||||
taskMgr.Register(tasks.NewChapterThumbnailBackfillTask(chapterBackfiller, 25))
|
||||
}
|
||||
@@ -1534,13 +1414,14 @@ func main() {
|
||||
taskMgr.Register(tasks.NewSyncWatchProvidersTask(watchProviderService))
|
||||
}
|
||||
requestReconcileSvc := mediarequests.NewService(
|
||||
mediarequests.NewRepository(deps.DB, deps.SecretCipher),
|
||||
mediarequests.NewRepository(deps.DB),
|
||||
nil,
|
||||
mediarequests.NewCatalogPresence(
|
||||
catalog.NewItemRepository(deps.DB),
|
||||
catalog.NewProviderIDRepository(deps.DB),
|
||||
),
|
||||
)
|
||||
requestReconcileSvc.SetSecretResolver(settingsRepo)
|
||||
requestReconcileSvc.SetFulfillmentAdapters(radarr.NewClient(nil), sonarr.NewClient(nil))
|
||||
if userStoreProvider != nil {
|
||||
reconcileResolver := access.NewResolver(
|
||||
@@ -1552,15 +1433,13 @@ func main() {
|
||||
}
|
||||
taskMgr.Register(tasks.NewReconcileRequestsTask(requestReconcileSvc, 100))
|
||||
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil && pluginService != nil && pluginInstallationStore != nil {
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB, deps.SecretCipher)
|
||||
if err := autoscanRepo.MarkInterruptedEvents(appCtx); err != nil {
|
||||
slog.Warn("autoscan: failed to mark interrupted polls", "err", err)
|
||||
}
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB)
|
||||
autoscanSvc := api.BuildAutoscanService(
|
||||
autoscanRepo,
|
||||
pluginService,
|
||||
pluginInstallationStore,
|
||||
mediarequests.NewRepository(deps.DB, deps.SecretCipher),
|
||||
mediarequests.NewRepository(deps.DB),
|
||||
settingsRepo,
|
||||
deps.FolderRepo,
|
||||
deps.LibraryScanQueue,
|
||||
deps.RedisClient,
|
||||
@@ -1584,7 +1463,7 @@ func main() {
|
||||
taskMgr.Register(tasks.NewRepairProviderIDIntegrityTask(metadata.NewProviderIDIntegrityRepairer(deps.DB), historyReconciler))
|
||||
taskMgr.Register(tasks.NewReconcileWatchHistoryTask(historyReconciler))
|
||||
taskMgr.Register(tasks.NewSyncPodcastFeedsTask(podcastfeed.New(), podcastfeed.NewDBStore(deps.DB)))
|
||||
if audiobookEnricher != nil {
|
||||
if audiobooksEnabled && audiobookEnricher != nil {
|
||||
taskMgr.Register(tasks.NewSyncAudiobookMetadataTask(audiobookEnricher))
|
||||
}
|
||||
if pluginInstallationStore != nil && pluginRuntimeConfigStore != nil && pluginService != nil {
|
||||
@@ -1607,7 +1486,7 @@ func main() {
|
||||
// available. Routes are mounted at the root level by NewRouter (not under
|
||||
// /api/v1/) so ABS clients resolve /login, /api/*, /abs/api/*, and
|
||||
// /abs/socket.io/* without path prefix hacks.
|
||||
if absCompatEnabled && deps.DB != nil {
|
||||
if audiobooksEnabled && deps.DB != nil {
|
||||
absUserRepo := auth.NewUserRepository(deps.DB)
|
||||
absSessionRepo := auth.NewSessionRepository(deps.DB)
|
||||
absJWTService := auth.NewJWTService(
|
||||
@@ -1880,7 +1759,6 @@ func main() {
|
||||
compatDeps := jellycompat.Dependencies{
|
||||
Config: cfg,
|
||||
DB: deps.DB,
|
||||
SecretCipher: dataCipher,
|
||||
ClientIPResolver: ipResolver,
|
||||
ProxyPool: deps.ProxyPool,
|
||||
TranscodePool: deps.TranscodePool,
|
||||
@@ -1934,7 +1812,7 @@ func main() {
|
||||
compatDeps.FileResolver = deps.FileRepo
|
||||
}
|
||||
|
||||
compatDeps.SubtitleRepo = subtitles.NewPgRepository(deps.DB, deps.SecretCipher)
|
||||
compatDeps.SubtitleRepo = subtitles.NewPgRepository(deps.DB)
|
||||
|
||||
// Construct auth service for jellycompat login.
|
||||
userRepo := auth.NewUserRepository(deps.DB)
|
||||
@@ -1950,17 +1828,47 @@ func main() {
|
||||
provider := auth.NewLocalProvider(userRepo, sessionRepo)
|
||||
compatDeps.AuthService = auth.NewService(provider, jwtService, sessionRepo, userRepo, nil, nil, nil)
|
||||
|
||||
// Access filter resolver for viewer-scoped library access.
|
||||
// Backed by the shared access.Resolver so account-level library
|
||||
// restrictions (users.library_ids), profile restrictions,
|
||||
// user-disabled libraries, and rating/quality ceilings apply to
|
||||
// the compat API exactly as they do to the native API.
|
||||
if userStoreProvider != nil {
|
||||
compatDeps.AccessFilterFn = jellycompat.NewScopeAccessFilter(access.NewResolver(
|
||||
userRepo,
|
||||
userStoreProvider,
|
||||
nil, // profile tokens unused: compat login already verifies PINs
|
||||
))
|
||||
// Access filter resolver for profile-scoped library access.
|
||||
compatDeps.AccessFilterFn = func(ctx context.Context, userID int, profileID string) catalog.AccessFilter {
|
||||
if userStoreProvider == nil {
|
||||
return catalog.AccessFilter{}
|
||||
}
|
||||
store, err := userStoreProvider.ForUser(ctx, userID)
|
||||
if err != nil {
|
||||
return catalog.AccessFilter{}
|
||||
}
|
||||
profile, err := store.GetProfile(ctx, profileID)
|
||||
if err != nil || profile == nil {
|
||||
return catalog.AccessFilter{}
|
||||
}
|
||||
filter := catalog.AccessFilter{
|
||||
MaxContentRating: profile.MaxContentRating,
|
||||
}
|
||||
if profile.LibraryRestrictionsEnabled && len(profile.AllowedLibraryIDs) > 0 {
|
||||
filter.AllowedLibraryIDs = profile.AllowedLibraryIDs
|
||||
}
|
||||
// Apply user-disabled library IDs.
|
||||
if raw, err := store.GetSetting(ctx, "disabled_library_ids"); err == nil && raw != "" {
|
||||
var disabled []int
|
||||
if json.Unmarshal([]byte(raw), &disabled) == nil && len(disabled) > 0 {
|
||||
if filter.AllowedLibraryIDs != nil {
|
||||
filtered := make([]int, 0, len(filter.AllowedLibraryIDs))
|
||||
disSet := make(map[int]struct{}, len(disabled))
|
||||
for _, id := range disabled {
|
||||
disSet[id] = struct{}{}
|
||||
}
|
||||
for _, id := range filter.AllowedLibraryIDs {
|
||||
if _, ok := disSet[id]; !ok {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
filter.AllowedLibraryIDs = filtered
|
||||
} else {
|
||||
filter.DisabledLibraryIDs = disabled
|
||||
}
|
||||
}
|
||||
}
|
||||
return filter
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2052,9 +1960,6 @@ func main() {
|
||||
case sig := <-sigCh:
|
||||
appCancel()
|
||||
slog.Info("received signal, shutting down", "signal", sig)
|
||||
case <-restartReqCh:
|
||||
appCancel()
|
||||
slog.Info("server restart requested, shutting down")
|
||||
case serverErr := <-errCh:
|
||||
appCancel()
|
||||
slog.Error("server error, shutting down", "error", serverErr)
|
||||
@@ -2224,198 +2129,6 @@ func configureS3Clients(cfg *config.Config, deps *api.Dependencies) {
|
||||
}
|
||||
}
|
||||
|
||||
type markerPluginCapabilityStore interface {
|
||||
ListEnabled(ctx context.Context) ([]*plugins.Installation, error)
|
||||
ListCapabilities(ctx context.Context, installationID int) ([]*plugins.Capability, error)
|
||||
}
|
||||
|
||||
type markerPluginRuntimeConfigStore interface {
|
||||
ListGlobalConfigs(ctx context.Context, installationID int) ([]*plugins.RuntimeConfig, error)
|
||||
PutGlobalConfig(ctx context.Context, installationID int, key string, value map[string]any) error
|
||||
}
|
||||
|
||||
type markerLegacySettingsStore interface {
|
||||
Get(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
func reloadMarkerPluginProviders(
|
||||
ctx context.Context,
|
||||
registry *markers.Registry,
|
||||
configStore *markers.ProviderConfigStore,
|
||||
store markerPluginCapabilityStore,
|
||||
runtimeConfigs markerPluginRuntimeConfigStore,
|
||||
legacySettings markerLegacySettingsStore,
|
||||
resolver *markers.PluginResolverAdapter,
|
||||
) error {
|
||||
if registry == nil {
|
||||
return nil
|
||||
}
|
||||
var providers []markers.Provider
|
||||
if store == nil || resolver == nil {
|
||||
return registry.SetProviders(providers)
|
||||
}
|
||||
|
||||
installations, err := store.ListEnabled(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list enabled plugin installations: %w", err)
|
||||
}
|
||||
sort.Slice(installations, func(i, j int) bool {
|
||||
if installations[i] == nil {
|
||||
return false
|
||||
}
|
||||
if installations[j] == nil {
|
||||
return true
|
||||
}
|
||||
return installations[i].ID < installations[j].ID
|
||||
})
|
||||
|
||||
nextPriority := 1000
|
||||
for _, installation := range installations {
|
||||
if installation == nil {
|
||||
continue
|
||||
}
|
||||
capabilities, err := store.ListCapabilities(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list marker provider capabilities for installation %d: %w", installation.ID, err)
|
||||
}
|
||||
sort.Slice(capabilities, func(i, j int) bool {
|
||||
if capabilities[i] == nil {
|
||||
return false
|
||||
}
|
||||
if capabilities[j] == nil {
|
||||
return true
|
||||
}
|
||||
return capabilities[i].ID < capabilities[j].ID
|
||||
})
|
||||
for _, capability := range capabilities {
|
||||
if capability == nil || capability.Type != sdkcapability.MarkerProvider {
|
||||
continue
|
||||
}
|
||||
descriptor, err := plugins.DecodeCapability(capability)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode marker provider capability %d/%s: %w", installation.ID, capability.ID, err)
|
||||
}
|
||||
metadataMap := markerCapabilityMetadata(descriptor)
|
||||
provider, err := markers.NewPluginProvider(markers.PluginProviderOptions{
|
||||
InstallationID: installation.ID,
|
||||
CapabilityID: capability.ID,
|
||||
DisplayName: firstNonEmptyMarkerText(descriptor.GetDisplayName(), capability.ID),
|
||||
PluginID: installation.PluginID,
|
||||
RequiredExternalIDs: markers.PluginRequiredExternalIDsFromMetadata(metadataMap),
|
||||
}, resolver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
providers = append(providers, provider)
|
||||
|
||||
priority := nextPriority
|
||||
nextPriority++
|
||||
if configuredPriority, ok := markers.PluginDefaultFetchPriorityFromMetadata(metadataMap); ok {
|
||||
priority = configuredPriority
|
||||
}
|
||||
if configStore != nil {
|
||||
defaultConfig := markers.ProviderConfig{
|
||||
Provider: provider.ID(),
|
||||
FetchEnabled: true,
|
||||
FetchPriority: priority,
|
||||
ContributeEnabled: false,
|
||||
ContributeAutoLocal: false,
|
||||
ContributeMinConfidence: 0.95,
|
||||
}
|
||||
if legacy, ok := legacyIntroDBProviderConfig(configStore, installation, capability, provider.ID()); ok {
|
||||
defaultConfig = legacy
|
||||
}
|
||||
if err := configStore.Ensure(ctx, defaultConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := copyLegacyIntroDBPluginConfig(ctx, runtimeConfigs, legacySettings, installation, capability); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return registry.SetProviders(providers)
|
||||
}
|
||||
|
||||
func legacyIntroDBProviderConfig(
|
||||
configStore *markers.ProviderConfigStore,
|
||||
installation *plugins.Installation,
|
||||
capability *plugins.Capability,
|
||||
providerID string,
|
||||
) (markers.ProviderConfig, bool) {
|
||||
if configStore == nil ||
|
||||
installation == nil ||
|
||||
capability == nil ||
|
||||
installation.PluginID != "silo.theintrodb" ||
|
||||
capability.ID != "introdb" {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
if _, exists := configStore.Get(providerID); exists {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
legacy, ok := configStore.Get("introdb")
|
||||
if !ok {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
legacy.Provider = providerID
|
||||
return legacy, true
|
||||
}
|
||||
|
||||
func copyLegacyIntroDBPluginConfig(
|
||||
ctx context.Context,
|
||||
runtimeConfigs markerPluginRuntimeConfigStore,
|
||||
legacySettings markerLegacySettingsStore,
|
||||
installation *plugins.Installation,
|
||||
capability *plugins.Capability,
|
||||
) error {
|
||||
if runtimeConfigs == nil ||
|
||||
legacySettings == nil ||
|
||||
installation == nil ||
|
||||
capability == nil ||
|
||||
installation.PluginID != "silo.theintrodb" ||
|
||||
capability.ID != "introdb" {
|
||||
return nil
|
||||
}
|
||||
configs, err := runtimeConfigs.ListGlobalConfigs(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list TheIntroDB plugin config: %w", err)
|
||||
}
|
||||
for _, config := range configs {
|
||||
if config != nil && config.Key == "account" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
apiKey, err := legacySettings.Get(ctx, "introdb.api_key")
|
||||
if err != nil {
|
||||
return fmt.Errorf("load legacy introdb.api_key: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(apiKey) == "" {
|
||||
return nil
|
||||
}
|
||||
if err := runtimeConfigs.PutGlobalConfig(ctx, installation.ID, "account", map[string]any{
|
||||
"api_key": strings.TrimSpace(apiKey),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("copy legacy introdb.api_key to plugin config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markerCapabilityMetadata(descriptor *pluginv1.CapabilityDescriptor) map[string]any {
|
||||
if descriptor == nil || descriptor.GetMetadata() == nil {
|
||||
return nil
|
||||
}
|
||||
return descriptor.GetMetadata().AsMap()
|
||||
}
|
||||
|
||||
func firstNonEmptyMarkerText(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mapFolderTypeToMediaType maps silo's MediaFolder.Type values
|
||||
// ("movies", "series", "mixed") to the SDK's MediaType values
|
||||
// ("movie", "tv", "mixed"). Unknown values map to "mixed".
|
||||
@@ -2434,7 +2147,7 @@ func mapFolderTypeToMediaType(t string) string {
|
||||
// exposes Get) to the audiobooks.SettingsReader interface (which
|
||||
// requires GetString). The two signatures are identical modulo name.
|
||||
type audiobooksSettingsAdapter struct {
|
||||
repo catalog.SettingsStore
|
||||
repo *catalog.ServerSettingsRepo
|
||||
}
|
||||
|
||||
func (a *audiobooksSettingsAdapter) GetString(ctx context.Context, key string) (string, error) {
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
)
|
||||
|
||||
type fakeMarkerRuntimeConfigs struct {
|
||||
configs map[int][]*plugins.RuntimeConfig
|
||||
puts []plugins.RuntimeConfig
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) ListGlobalConfigs(_ context.Context, installationID int) ([]*plugins.RuntimeConfig, error) {
|
||||
return append([]*plugins.RuntimeConfig(nil), f.configs[installationID]...), nil
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) PutGlobalConfig(_ context.Context, installationID int, key string, value map[string]any) error {
|
||||
f.puts = append(f.puts, plugins.RuntimeConfig{InstallationID: installationID, Key: key, Value: value})
|
||||
f.configs[installationID] = append(f.configs[installationID], &plugins.RuntimeConfig{
|
||||
InstallationID: installationID,
|
||||
Key: key,
|
||||
Value: value,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeMarkerLegacySettings map[string]string
|
||||
|
||||
func (f fakeMarkerLegacySettings) Get(_ context.Context, key string) (string, error) {
|
||||
return f[key], nil
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigCopiesAPIKeyOnce(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
installation := &plugins.Installation{ID: 42, PluginID: "silo.theintrodb"}
|
||||
capability := &plugins.Capability{ID: "introdb"}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": " legacy-key "},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
got := runtimeConfigs.puts[0]
|
||||
if got.InstallationID != 42 || got.Key != "account" || got.Value["api_key"] != "legacy-key" {
|
||||
t.Fatalf("put = %+v, want account api_key copy", got)
|
||||
}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "new-key"},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("second copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("second copy overwrote config; puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigIgnoresOtherPlugins(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "legacy-key"},
|
||||
&plugins.Installation{ID: 7, PluginID: "silo.other"},
|
||||
&plugins.Capability{ID: "introdb"},
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 0 {
|
||||
t.Fatalf("puts = %d, want 0 for non-TheIntroDB plugin", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
services:
|
||||
silo:
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: ${NVIDIA_GPU_COUNT:-1}
|
||||
capabilities: [gpu]
|
||||
|
||||
# Optional distributed-mode example. Most single-host installs should
|
||||
# just use the integrated service above.
|
||||
# silo-transcode:
|
||||
# deploy:
|
||||
# resources:
|
||||
# reservations:
|
||||
# devices:
|
||||
# - driver: nvidia
|
||||
# count: ${NVIDIA_GPU_COUNT:-1}
|
||||
# capabilities: [gpu]
|
||||
+1
-10
@@ -36,11 +36,6 @@ services:
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MODE: integrated
|
||||
# Master key for at-rest credential encryption. Required — the server
|
||||
# refuses to start without it. Generate with: openssl rand -base64 48
|
||||
# Back it up SEPARATELY from your database dumps; losing it makes encrypted
|
||||
# secrets unrecoverable. See docs/architecture/secret-encryption.md.
|
||||
SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env — generate one with openssl rand -base64 48}
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins
|
||||
@@ -51,7 +46,7 @@ services:
|
||||
- "${ABS_PORT:-13378}:13378"
|
||||
volumes:
|
||||
- ${MEDIA_ROOT:?Set MEDIA_ROOT in .env to the host media path}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro
|
||||
- ${MEDIA_BOOKS_ROOT:-${MEDIA_ROOT}}:${MEDIA_BOOKS_CONTAINER_ROOT:-${MEDIA_CONTAINER_ROOT:-/mnt/media}/books}:ro
|
||||
- ${MEDIA_BOOKS_ROOT:-${MEDIA_ROOT}}:${MEDIA_BOOKS_CONTAINER_ROOT:-${MEDIA_CONTAINER_ROOT:-/mnt/media}}:ro
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/plugins:/var/lib/silo/plugins
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro
|
||||
@@ -76,8 +71,6 @@ services:
|
||||
# profiles: [proxy]
|
||||
# environment:
|
||||
# MODE: proxy
|
||||
# # Must be the SAME SECRET_KEY as the primary node (shared encrypted data).
|
||||
# SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env — generate one with openssl rand -base64 48}
|
||||
# DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
# REDIS_URL: redis://redis:6379
|
||||
# PORT: "8080"
|
||||
@@ -102,8 +95,6 @@ services:
|
||||
# profiles: [transcode]
|
||||
# environment:
|
||||
# MODE: transcode
|
||||
# # Must be the SAME SECRET_KEY as the primary node (shared encrypted data).
|
||||
# SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env — generate one with openssl rand -base64 48}
|
||||
# DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
# REDIS_URL: redis://redis:6379
|
||||
# SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
# At-rest credential encryption
|
||||
|
||||
Silo encrypts third-party integration credentials and other server-owned secrets
|
||||
at rest. Sonarr/Radarr API keys, S3 keys, watch-sync OAuth tokens, the
|
||||
Audiobookshelf-compat signing key, and the sensitive `server_settings` entries
|
||||
are stored as ciphertext in PostgreSQL and decrypted only in memory. The key that
|
||||
protects them lives **outside** the database, so a full DB dump or compromise
|
||||
does not expose the secrets.
|
||||
|
||||
## How it works
|
||||
|
||||
- **Cipher:** AES-256-GCM with a random 12-byte nonce per value. Implemented in
|
||||
`internal/secret`.
|
||||
- **Key derivation:** the data key is HKDF-SHA256–derived from the `SECRET_KEY`
|
||||
environment variable with a versioned domain label. `SECRET_KEY` is the
|
||||
encryption root; it is never stored in the database.
|
||||
- **Envelope:** every ciphertext is stored as `enc:v1:<base64url(nonce‖sealed)>`.
|
||||
The `enc:v1:` prefix is the version marker (see *Key rotation* below).
|
||||
- **Row binding (AAD):** each ciphertext is GCM-bound to its logical row via
|
||||
additional-authenticated data — `table:column:<pk>` for per-row columns,
|
||||
`server_settings:<key>` for settings. A DB-write attacker therefore cannot move
|
||||
a credential blob from one row/column to another: decrypting under a different
|
||||
binding fails authentication.
|
||||
- **Read path:** an empty value stays empty; a non-`enc:v1:` value is treated as
|
||||
legacy plaintext and passed through unchanged (so a not-yet-migrated row keeps
|
||||
working); an `enc:v1:` value is decrypted and **any** failure (wrong key,
|
||||
tampering, truncation) is surfaced as an error — it is never silently used as a
|
||||
credential.
|
||||
|
||||
## SECRET_KEY
|
||||
|
||||
`SECRET_KEY` is **required**. The server refuses to start without it (it fatals
|
||||
in `config.LoadBootstrap`), and it must be at least 32 characters.
|
||||
|
||||
Generate one with:
|
||||
|
||||
```bash
|
||||
openssl rand -base64 48
|
||||
```
|
||||
|
||||
Set it in the deployment environment (or in `.env` for source/dev runs, which
|
||||
`godotenv` loads at startup). See `.env.example`.
|
||||
|
||||
### Back it up — separately from the database
|
||||
|
||||
Treat `SECRET_KEY` like a CA private key:
|
||||
|
||||
- **Store it outside your database backups.** A DB dump contains only ciphertext;
|
||||
the value of at-rest encryption is entirely lost if the key travels with the
|
||||
dump.
|
||||
- Keep it in a secrets manager / sealed secret / password manager, not in the
|
||||
repo or in the same bucket as your Postgres backups.
|
||||
- Rotating the *deployment* (new host, restored DB) requires the **same**
|
||||
`SECRET_KEY`. Restoring a database backup onto a node with a different
|
||||
`SECRET_KEY` makes every encrypted secret unreadable.
|
||||
|
||||
### Key loss
|
||||
|
||||
If `SECRET_KEY` is lost, the encrypted secrets are **unrecoverable** — that is the
|
||||
point of keeping the key out of the database. Recovery means re-entering the
|
||||
affected credentials:
|
||||
|
||||
- Re-enter Sonarr/Radarr and Autoscan API keys, S3 keys, watch-sync connections,
|
||||
history-import tokens, and subtitle provider credentials.
|
||||
- `auth.jwt_secret` becomes unreadable, so all existing sessions are invalid and
|
||||
users must log in again (a new secret is generated on next boot if the row is
|
||||
cleared — see below).
|
||||
|
||||
## Rollout (automatic backfill)
|
||||
|
||||
On the first boot after deploying this change, the primary (migration-running)
|
||||
node runs an idempotent, best-effort startup backfill that encrypts any
|
||||
remaining plaintext in place:
|
||||
|
||||
1. sensitive `server_settings` keys,
|
||||
2. the per-table credential columns (subtitles, watch-sync, webhook-sync,
|
||||
history-import, including temporary server-list credentials stored in
|
||||
session JSON),
|
||||
3. the two arr `api_key_ref` columns — these are **resolved-then-encrypted**: a
|
||||
legacy row that held a `server_settings` reference (e.g.
|
||||
`requests.radarr.api_key`) is collapsed to the real credential before being
|
||||
encrypted.
|
||||
|
||||
The backfill is safe to run repeatedly: already-encrypted values are skipped, and
|
||||
a per-row guard makes concurrent multi-node boots converge without
|
||||
double-encrypting. A failed row is logged and left as plaintext (no new exposure)
|
||||
rather than blocking boot. Proxy/transcode nodes skip the backfill; they read
|
||||
whatever the primary encrypted. No manual steps are required.
|
||||
|
||||
## Rollback / downgrade hazard
|
||||
|
||||
Downgrading to a binary that predates this change is **not** safe while secrets
|
||||
are encrypted, because the old binary has no read path: it would read
|
||||
`enc:v1:auth.jwt_secret` as a literal JWT secret (invalidating all sessions) and
|
||||
read `enc:v1:`-prefixed integration keys as garbage credentials.
|
||||
|
||||
To downgrade safely you must first return the affected values to plaintext, for
|
||||
example:
|
||||
|
||||
- Decrypt the sensitive `server_settings` back to plaintext (run a one-off that
|
||||
reads each via the encrypting repo and writes the plaintext via the raw repo),
|
||||
**or**
|
||||
- At minimum, clear `auth.jwt_secret` so the older binary regenerates a fresh
|
||||
plaintext one (this still logs everyone out), and re-enter any integration
|
||||
credentials as plaintext.
|
||||
|
||||
There is no automatic "decrypt everything" downgrade path in this release; plan
|
||||
downgrades accordingly.
|
||||
|
||||
## Key rotation (future)
|
||||
|
||||
The `enc:v1:` envelope and the HKDF domain label are versioned. A future rotation
|
||||
would introduce `enc:v2:` with a new derived key (and `Decrypt` already dispatches
|
||||
on the version, returning an explicit error for an unknown version). Rotation is
|
||||
out of scope for this release; the versioning exists so it can be added without a
|
||||
data migration of the envelope format.
|
||||
|
||||
## Scope and known gaps
|
||||
|
||||
Covered: arr (Requests + Autoscan) API keys, S3 keys, all sensitive
|
||||
`server_settings`, watch-sync tokens, webhook-sync `access_token`,
|
||||
history-import admin/session tokens and temporary server-list credentials,
|
||||
subtitle provider `api_key`/`password`,
|
||||
the Jellyfin-compat session's bridged Silo access/refresh tokens
|
||||
(`jellycompat_sessions.streamapp_access_token` / `streamapp_refresh_token`), and
|
||||
the ABS signing key.
|
||||
|
||||
Deliberately **not** encrypted (tracked as follow-ups):
|
||||
|
||||
- **Equality-looked-up secrets** — these are matched by exact value
|
||||
(`WHERE … = $1`), and AES-GCM is randomized, so encrypting them would break the
|
||||
lookup. They need a deterministic **blind-index hash** column instead:
|
||||
`api_keys.api_key`, `webhook_sync_connections.webhook_secret`,
|
||||
`jellycompat_sessions.token`, and `watch_together_rooms.join_token`.
|
||||
- **Plugin runtime config** — `plugin_runtime_configs.config_value` is opaque
|
||||
plugin-defined JSONB whose secret fields are manifest-declared and whose runtime
|
||||
lives in a separate repo; encrypting it correctly needs a coordinated,
|
||||
manifest-aware design.
|
||||
|
||||
Excluded (not a gap): `plex_sync_connections.*` is a dead table (zero Go
|
||||
references); `oauth_completion.token_ciphertext` is already AES-GCM;
|
||||
`users.password_hash` and the `*_hash` columns are already hashed.
|
||||
@@ -1,145 +0,0 @@
|
||||
# First-Party Audio API
|
||||
|
||||
This API is the Silo-native contract for audiobook playback. Clients should use
|
||||
it for Silo first-party audio experiences instead of the Audiobookshelf
|
||||
compatibility API. Music can reuse the same session, track, and progress
|
||||
concepts when the music UX lands.
|
||||
|
||||
## Authentication
|
||||
|
||||
All endpoints require normal Silo user authentication. Playback start, sync,
|
||||
close, and bookmark writes also require the active profile headers used by the
|
||||
first-party clients.
|
||||
|
||||
Track stream URLs are returned from the start response and are scoped to the
|
||||
created audio session. The server validates that the session still belongs to
|
||||
the authenticated user, and to the active profile when a profile is attached.
|
||||
|
||||
## Start Playback
|
||||
|
||||
`POST /api/v1/audio/playback/start`
|
||||
|
||||
Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"content_id": "128814341867700228",
|
||||
"start_position": 120,
|
||||
"restart": false
|
||||
}
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{
|
||||
"session_id": "uuid",
|
||||
"content_id": "128814341867700228",
|
||||
"type": "audiobook",
|
||||
"title": "Book Title",
|
||||
"subtitle": "Author or series subtitle",
|
||||
"poster_url": "https://server/api/v1/images/...",
|
||||
"backdrop_url": "https://server/api/v1/images/...",
|
||||
"total_duration_seconds": 8746,
|
||||
"resume_position_seconds": 120,
|
||||
"tracks": [
|
||||
{
|
||||
"index": 0,
|
||||
"file_id": 123,
|
||||
"file_name": "Part 1.mp3",
|
||||
"duration_seconds": 3600,
|
||||
"start_offset_seconds": 0,
|
||||
"stream_url": "/api/v1/audio/playback/{session_id}/tracks/0",
|
||||
"stream_type": "progressive",
|
||||
"play_method": "direct",
|
||||
"codec": "mp3",
|
||||
"container": "mp3",
|
||||
"bitrate": 128000,
|
||||
"chapters": []
|
||||
}
|
||||
],
|
||||
"chapters": [
|
||||
{
|
||||
"index": 0,
|
||||
"title": "Chapter 1",
|
||||
"start_seconds": 0,
|
||||
"end_seconds": 1800,
|
||||
"track_index": 0
|
||||
}
|
||||
],
|
||||
"bookmarks": []
|
||||
}
|
||||
```
|
||||
|
||||
`total_duration_seconds`, `resume_position_seconds`, global chapter times, and
|
||||
sync positions are cumulative book seconds, not per-file seconds. Clients map
|
||||
global time to the active track by finding the last track whose
|
||||
`start_offset_seconds` is less than or equal to the target time.
|
||||
|
||||
Tracks are ordered by presentation part index when available, then by path and
|
||||
file id. The server direct-plays AVFoundation-safe formats and returns
|
||||
`play_method: "aac"` for fallback streams that are remuxed/transcoded to an
|
||||
Apple-safe AAC container.
|
||||
|
||||
## Stream Track
|
||||
|
||||
`GET /api/v1/audio/playback/{session_id}/tracks/{track_index}`
|
||||
|
||||
The response is the media stream for the selected track. Clients should request
|
||||
the track for the current global time and seek within that track using:
|
||||
|
||||
`local_seconds = global_seconds - track.start_offset_seconds`
|
||||
|
||||
## Sync Progress
|
||||
|
||||
`PATCH /api/v1/audio/playback/{session_id}/sync`
|
||||
|
||||
Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"position": 1234,
|
||||
"duration": 8746,
|
||||
"is_paused": false
|
||||
}
|
||||
```
|
||||
|
||||
The server persists `position` as the audiobook progress position. The client
|
||||
should send cumulative book seconds on a throttle while playing, on manual
|
||||
seeks, and once more before closing the session.
|
||||
|
||||
## Close Playback
|
||||
|
||||
`POST /api/v1/audio/playback/{session_id}/close`
|
||||
|
||||
Request:
|
||||
|
||||
```json
|
||||
{
|
||||
"position": 1234,
|
||||
"duration": 8746,
|
||||
"is_paused": true
|
||||
}
|
||||
```
|
||||
|
||||
Close performs a final progress sync and expires the in-memory audio session.
|
||||
|
||||
## Bookmarks
|
||||
|
||||
`GET /api/v1/audio/items/{content_id}/bookmarks`
|
||||
|
||||
`POST /api/v1/audio/items/{content_id}/bookmarks`
|
||||
|
||||
`DELETE /api/v1/audio/items/{content_id}/bookmarks/{time_seconds}`
|
||||
|
||||
Create request:
|
||||
|
||||
```json
|
||||
{
|
||||
"time_seconds": 1234,
|
||||
"title": "Chapter note"
|
||||
}
|
||||
```
|
||||
|
||||
Bookmark positions are cumulative book seconds so they remain stable across
|
||||
multi-file audiobooks and client platforms.
|
||||
@@ -17,14 +17,14 @@
|
||||
| Path | Created/Modified | Purpose |
|
||||
|---|---|---|
|
||||
| `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` | Create | Findings doc produced by Task 1. Locks the remaining schema/code decisions. |
|
||||
| `migrations/147_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/147_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/157_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/157_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/159_media_folders_kind_noop.up.sql` | Create (conditional — see Task 4 step 1) | Document `media_folders.type='audiobooks'`; no column change needed. |
|
||||
| `migrations/159_media_folders_kind_noop.down.sql` | Create (same condition) | No-op rollback. |
|
||||
| `migrations/160_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/160_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `migrations/139_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/139_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/140_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/140_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/141_media_libraries_kind.up.sql` | Create (conditional — see Task 4 step 1) | Add `kind` column to `media_libraries`. |
|
||||
| `migrations/141_media_libraries_kind.down.sql` | Create (same condition) | Drop the column. |
|
||||
| `migrations/142_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/142_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `internal/audiobooks/doc.go` | Create | Package comment + intent. |
|
||||
| `internal/audiobooks/service.go` | Create | Empty `Service` struct, `New` constructor, `Enabled()` reader. |
|
||||
| `internal/audiobooks/service_test.go` | Create | One unit test proving the package compiles and `Enabled()` returns the setting value. |
|
||||
@@ -240,15 +240,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Migration 147 — `abs_sessions`
|
||||
## Task 2: Migration 139 — `abs_sessions`
|
||||
|
||||
Parallel of `jellycompat_sessions` for Audiobookshelf clients. Lets ABS
|
||||
mobile/desktop apps reconnect without re-authenticating against silo's
|
||||
main `auth_sessions`.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/147_abs_sessions.up.sql`
|
||||
- Create: `migrations/147_abs_sessions.down.sql`
|
||||
- Create: `migrations/139_abs_sessions.up.sql`
|
||||
- Create: `migrations/139_abs_sessions.down.sql`
|
||||
|
||||
> **Note:** If Task 1, Step 1.2 found the next number is not 139, rename
|
||||
> both filenames to match. All other tasks downstream of 139 shift by the
|
||||
@@ -266,7 +266,7 @@ main `auth_sessions`.
|
||||
|
||||
### Step 2.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/147_abs_sessions.up.sql` with:
|
||||
- [ ] **Create** `migrations/139_abs_sessions.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Audiobookshelf-compatible client sessions. Parallel to
|
||||
@@ -277,7 +277,7 @@ main `auth_sessions`.
|
||||
CREATE TABLE IF NOT EXISTS public.abs_sessions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
device_name TEXT,
|
||||
client_name TEXT,
|
||||
@@ -287,8 +287,8 @@ main `auth_sessions`.
|
||||
revoked_at TIMESTAMP WITH TIME ZONE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token_hash
|
||||
ON public.abs_sessions (token_hash);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token
|
||||
ON public.abs_sessions (token);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_abs_sessions_user_device
|
||||
ON public.abs_sessions (user_id, device_id);
|
||||
@@ -299,12 +299,12 @@ main `auth_sessions`.
|
||||
|
||||
### Step 2.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/147_abs_sessions.down.sql` with:
|
||||
- [ ] **Create** `migrations/139_abs_sessions.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_last_seen;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_user_device;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token_hash;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token;
|
||||
DROP TABLE IF EXISTS public.abs_sessions;
|
||||
```
|
||||
|
||||
@@ -328,27 +328,27 @@ main `auth_sessions`.
|
||||
- [ ] **Run** (each command should now succeed):
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 147;"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 139;"
|
||||
```
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=147`.
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=139`.
|
||||
|
||||
- [ ] **Run** a sample insert/delete to prove constraints work:
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
INSERT INTO abs_sessions (user_id, token_hash, device_id, device_name, client_name, client_version)
|
||||
VALUES ((SELECT id FROM users ORDER BY id LIMIT 1), 'test-token-hash-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token_hash = 'test-token-hash-001';
|
||||
INSERT INTO abs_sessions (user_id, token, device_id, device_name, client_name, client_version)
|
||||
VALUES (1, 'test-token-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token = 'test-token-001';
|
||||
"
|
||||
```
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (At least one user must exist.)
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (User id 1 must exist — silo's first user.)
|
||||
|
||||
### Step 2.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/147_abs_sessions.up.sql migrations/147_abs_sessions.down.sql
|
||||
git add migrations/139_abs_sessions.up.sql migrations/139_abs_sessions.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 147 add abs_sessions table
|
||||
feat(audiobooks): migration 139 add abs_sessions table
|
||||
|
||||
Parallel of jellycompat_sessions for Audiobookshelf-compatible clients.
|
||||
Lets ABS mobile/desktop apps maintain a device-bound session that
|
||||
@@ -361,15 +361,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Migration 157 — `podcast_feeds`
|
||||
## Task 3: Migration 140 — `podcast_feeds`
|
||||
|
||||
One row per subscribed podcast. The podcast itself lives in `media_items`
|
||||
(`type='podcast'`); this side table carries the RSS-specific metadata the
|
||||
feed refresher needs.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/157_podcast_feeds.up.sql`
|
||||
- Create: `migrations/157_podcast_feeds.down.sql`
|
||||
- Create: `migrations/140_podcast_feeds.up.sql`
|
||||
- Create: `migrations/140_podcast_feeds.down.sql`
|
||||
|
||||
### Step 3.1: Write the failing assertion
|
||||
|
||||
@@ -381,7 +381,7 @@ feed refresher needs.
|
||||
|
||||
### Step 3.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.up.sql` with:
|
||||
- [ ] **Create** `migrations/140_podcast_feeds.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- RSS-feed metadata for subscribed podcasts. One row per podcast
|
||||
@@ -416,7 +416,7 @@ feed refresher needs.
|
||||
|
||||
### Step 3.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.down.sql` with:
|
||||
- [ ] **Create** `migrations/140_podcast_feeds.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_podcast_feeds_due_for_refresh;
|
||||
@@ -438,17 +438,17 @@ feed refresher needs.
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 157;"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 140;"
|
||||
```
|
||||
Expected: column list printed; one row with `version=157`.
|
||||
Expected: column list printed; one row with `version=140`.
|
||||
|
||||
### Step 3.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/157_podcast_feeds.up.sql migrations/157_podcast_feeds.down.sql
|
||||
git add migrations/140_podcast_feeds.up.sql migrations/140_podcast_feeds.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 157 add podcast_feeds table
|
||||
feat(audiobooks): migration 140 add podcast_feeds table
|
||||
|
||||
Side table on media_items for RSS-subscribed podcasts. Holds feed URL,
|
||||
ETag/Last-Modified for conditional fetches, last-refresh timestamp, and
|
||||
@@ -462,7 +462,7 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Migration 159 — `media_libraries.kind` audiobook value (no-op)
|
||||
## Task 4: Migration 141 — `media_libraries.kind` (conditional)
|
||||
|
||||
Adds a discriminator column so the scanner can branch on
|
||||
`audiobooks`/`podcasts` libraries without filename heuristics.
|
||||
@@ -474,8 +474,8 @@ Adds a discriminator column so the scanner can branch on
|
||||
> Otherwise, proceed with the steps below.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/159_media_folders_kind_noop.up.sql`
|
||||
- Create: `migrations/159_media_folders_kind_noop.down.sql`
|
||||
- Create: `migrations/141_media_libraries_kind.up.sql`
|
||||
- Create: `migrations/141_media_libraries_kind.down.sql`
|
||||
|
||||
### Step 4.1: Write the failing assertion
|
||||
|
||||
@@ -487,7 +487,7 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
### Step 4.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.up.sql` with:
|
||||
- [ ] **Create** `migrations/141_media_libraries_kind.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Per-library content discriminator. Lets the scanner choose the right
|
||||
@@ -509,7 +509,7 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
### Step 4.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.down.sql` with:
|
||||
- [ ] **Create** `migrations/141_media_libraries_kind.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_media_libraries_kind;
|
||||
@@ -551,9 +551,9 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/159_media_folders_kind_noop.up.sql migrations/159_media_folders_kind_noop.down.sql
|
||||
git add migrations/141_media_libraries_kind.up.sql migrations/141_media_libraries_kind.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 159 document audiobooks media folder type
|
||||
feat(audiobooks): migration 141 add media_libraries.kind
|
||||
|
||||
Per-library discriminator that lets the scanner pick the right parser
|
||||
('movies', 'tv', 'audiobooks', 'podcasts') instead of inferring from
|
||||
@@ -567,15 +567,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Migration 160 — `audiobooks.enabled` feature flag
|
||||
## Task 5: Migration 142 — `audiobooks.enabled` feature flag
|
||||
|
||||
Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
"audiobooks compiled in but turned off" vs "fully live". Default is
|
||||
`false` so this sub-plan's landing is a strict no-op for users.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/160_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/160_audiobooks_feature_flag.down.sql`
|
||||
- Create: `migrations/142_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/142_audiobooks_feature_flag.down.sql`
|
||||
|
||||
### Step 5.1: Write the failing assertion
|
||||
|
||||
@@ -589,11 +589,11 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
### Step 5.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.up.sql` with:
|
||||
- [ ] **Create** `migrations/142_audiobooks_feature_flag.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Master kill-switch for the absorbed audiobooks feature. Defaults to
|
||||
-- 'false' so landing migrations 147/157/159/160 is a no-op for users; operators
|
||||
-- 'false' so landing migrations 139-142 is a no-op for users; operators
|
||||
-- flip this to 'true' at cutover.
|
||||
|
||||
INSERT INTO server_settings (key, value) VALUES ('audiobooks.enabled', 'false')
|
||||
@@ -602,7 +602,7 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
### Step 5.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.down.sql` with:
|
||||
- [ ] **Create** `migrations/142_audiobooks_feature_flag.down.sql` with:
|
||||
|
||||
```sql
|
||||
DELETE FROM server_settings WHERE key = 'audiobooks.enabled';
|
||||
@@ -625,9 +625,9 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/160_audiobooks_feature_flag.up.sql migrations/160_audiobooks_feature_flag.down.sql
|
||||
git add migrations/142_audiobooks_feature_flag.up.sql migrations/142_audiobooks_feature_flag.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 160 add audiobooks.enabled flag
|
||||
feat(audiobooks): migration 142 add audiobooks.enabled flag
|
||||
|
||||
Server-settings row that gates the absorbed audiobooks feature.
|
||||
Defaults to 'false' so sub-plan 1 lands as a strict no-op; subsequent
|
||||
@@ -646,7 +646,7 @@ EOF
|
||||
Creates an empty but compiling Go package, wires its constructor into
|
||||
`cmd/silo/main.go` so the package is referenced from the binary, and
|
||||
ships a unit test that proves the `Enabled()` reader returns the flag
|
||||
value seeded by migration 160.
|
||||
value seeded by migration 142.
|
||||
|
||||
This task proves the package compiles inside the rest of silo without
|
||||
changing any runtime behavior. No routes are mounted. No scheduled
|
||||
@@ -721,7 +721,7 @@ tasks are registered. No DB writes happen.
|
||||
|
||||
### Step 6.2: Run the test to verify it fails
|
||||
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/...`
|
||||
|
||||
Expected: build failure with messages like `undefined: New` and
|
||||
`package audiobooks not found`.
|
||||
@@ -777,9 +777,9 @@ tasks are registered. No DB writes happen.
|
||||
return &Service{settings: settings}
|
||||
}
|
||||
|
||||
// Enabled reports whether the audiobooks feature flag (set by
|
||||
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
|
||||
// Any value other than the literal string "true" reads as false; this matches how silo
|
||||
// Enabled reports whether the audiobooks feature flag (set by migration
|
||||
// 142 and toggled by operators) is currently true. Any value other than
|
||||
// the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
@@ -795,7 +795,7 @@ tasks are registered. No DB writes happen.
|
||||
|
||||
### Step 6.5: Run the test to verify it passes
|
||||
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/...`
|
||||
Expected: `ok github.com/Silo-Server/silo-server/internal/audiobooks ...`
|
||||
|
||||
### Step 6.6: Wire the constructor into `cmd/silo/main.go`
|
||||
@@ -831,13 +831,13 @@ are constructed and pass a settings store to.
|
||||
|
||||
### Step 6.7: Verify everything still compiles and tests pass
|
||||
|
||||
- [ ] **Run:** `go build ./...`
|
||||
- [ ] **Run:** `cd /opt/silo-server && go build ./...`
|
||||
Expected: exit 0, no output.
|
||||
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/... ./cmd/...`
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/... ./cmd/...`
|
||||
Expected: all passes.
|
||||
|
||||
- [ ] **Run:** `make lint`
|
||||
- [ ] **Run:** `cd /opt/silo-server && make lint`
|
||||
Expected: no new lint findings introduced by this task.
|
||||
|
||||
### Step 6.8: Smoke-test the running binary
|
||||
@@ -883,9 +883,9 @@ EOF
|
||||
| Spec section | Task(s) that cover it |
|
||||
|---|---|
|
||||
| Data model: reused tables | Task 1 (verification only) |
|
||||
| Data model: `abs_sessions` (migration 147) | Task 2 |
|
||||
| Data model: `podcast_feeds` (migration 157) | Task 3 |
|
||||
| Data model: `media_folders.type` audiobook value (migration 159 no-op) | Task 4 |
|
||||
| Data model: `abs_sessions` (mig 134→139) | Task 2 |
|
||||
| Data model: `podcast_feeds` (mig 135→140) | Task 3 |
|
||||
| Data model: `media_libraries.kind` (mig 136→141) | Task 4 |
|
||||
| Data model: data migration = none | n/a — confirmed in spec |
|
||||
| Feature flag (`audiobooks.enabled`) | Task 5 |
|
||||
| Architecture: `internal/audiobooks/` package | Task 6 |
|
||||
@@ -909,8 +909,9 @@ fires).
|
||||
test matches the production interface defined in `service.go`. Method
|
||||
name `GetString`, signature `(ctx context.Context, key string) (string, error)` — consistent across both files.
|
||||
|
||||
**Note on migration numbering:** Tasks 2–5 originally used numbers `139..142`
|
||||
based on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
**Note on migration numbering:** Tasks 2–5 use numbers `139..142` based
|
||||
on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. Task 1 Step 1.2 re-verifies this; if
|
||||
the executing agent finds the highest existing number has moved, it
|
||||
must shift filenames in Tasks 2–5 by the same delta and update the
|
||||
matching task headings and commit-message references.
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make silo's existing scanner recognize audiobook + podcast libraries (`media_folders.type='audiobooks'` or `'podcasts'`), walk them to find audio files, and write `media_items.type='audiobook'` / `media_items.type='podcast'` rows with chapter metadata and author/narrator role links. No external metadata enrichment runs in this sub-plan; the scanner extracts local tags and identity hints, and the audiobook metadata plugin/provider enriches canonical metadata later.
|
||||
**Goal:** Make silo's existing scanner recognize audiobook + podcast libraries (`media_folders.type='audiobooks'` or `'podcasts'`), walk them to find audio files, and write `media_items.type='audiobook'` / `media_items.type='podcast'` rows with chapter metadata and author/narrator role links. No external metadata enrichment in this sub-plan — file tags only.
|
||||
|
||||
**Architecture:** New constants (`PersonKindAuthor=7`, `PersonKindNarrator=8`, `audioExtensions`). New helpers (`isAudiobookLibraryType`, `isPodcastLibraryType`, `SupportsAudioFile`). New parser file `internal/scanner/audiobook.go` for the folder-to-media-items mapping. The chapter extraction reuses the existing `probe.go` ffprobe helper. Author/narrator names are upserted into the existing `people` and `item_people` tables via the same code paths used for movie actors. Matching/enrichment queue integration for the audiobook metadata plugin/provider is deferred.
|
||||
**Architecture:** New constants (`PersonKindAuthor=7`, `PersonKindNarrator=8`, `audioExtensions`). New helpers (`isAudiobookLibraryType`, `isPodcastLibraryType`, `SupportsAudioFile`). New parser file `internal/scanner/audiobook.go` for the folder-to-media-items mapping. The chapter extraction reuses the existing `probe.go` ffprobe helper. Author/narrator names are upserted into the existing `people` and `item_people` tables via the same code paths used for movie actors. No changes to the matching/enrichment metadata queues — those are deferred.
|
||||
|
||||
**Tech Stack:** Go 1.26 + pgx; ffprobe via `os/exec` (already vendored as `internal/scanner/probe.go`); ID3/MP4 tag parsing — verify what silo already vendors before introducing a new library (acceptable: `github.com/dhowden/tag` if not already present, OR shell out to ffprobe which exposes tag dictionaries).
|
||||
|
||||
@@ -408,8 +408,8 @@ ffmpeg -y -f lavfi -i "anullsrc=r=22050:cl=mono" -t 10 -metadata title="Test Aud
|
||||
-metadata artist="Test Author" -metadata album="Test Series" \
|
||||
-metadata:s:a:0 language=eng \
|
||||
internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a
|
||||
# Add chapters via a repo-local metadata fixture
|
||||
cat > internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta <<'EOF'
|
||||
# Add chapters via a temporary metadata file
|
||||
cat > /tmp/chapters.ffmeta <<'EOF'
|
||||
;FFMETADATA1
|
||||
[CHAPTER]
|
||||
TIMEBASE=1/1000
|
||||
@@ -423,9 +423,9 @@ END=10000
|
||||
title=Outro
|
||||
EOF
|
||||
ffmpeg -y -i internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a \
|
||||
-i internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta -map_metadata 1 -codec copy \
|
||||
-i /tmp/chapters.ffmeta -map_metadata 1 -codec copy \
|
||||
internal/scanner/testdata/audiobook_fixtures/single_book/book.m4b
|
||||
rm internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta
|
||||
rm internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a /tmp/chapters.ffmeta
|
||||
```
|
||||
|
||||
Add a test that calls the chapter-extraction function on `book.m4b` and asserts:
|
||||
@@ -1008,6 +1008,7 @@ Verify the full sub-plan landed cleanly.
|
||||
### Step 10.1: Full tests + build
|
||||
|
||||
```bash
|
||||
cd /opt/silo-server
|
||||
go build ./...
|
||||
go test ./internal/scanner/... ./internal/models/... ./internal/audiobooks/...
|
||||
go vet ./...
|
||||
@@ -1018,7 +1019,7 @@ All green.
|
||||
### Step 10.2: Rebuild + recreate silo container
|
||||
|
||||
```bash
|
||||
sudo docker build -t silo:latest .
|
||||
sudo docker build -t silo:latest /opt/silo-server
|
||||
sudo docker compose -p silo-prod up -d --force-recreate silo
|
||||
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
|
||||
```
|
||||
@@ -1064,7 +1065,7 @@ Otherwise: no commit needed.
|
||||
| Multi-file audiobooks | T7 |
|
||||
| Filesystem podcasts | T9 |
|
||||
| RSS podcasts | deferred to sub-plan 5 |
|
||||
| External enrichment through audiobook metadata plugin/provider | deferred per spec |
|
||||
| External enrichment (OpenLibrary/Google Books) | dropped per spec |
|
||||
|
||||
**Placeholder scan:** Task 8 has language like "grep for upsertMediaItem or similar in scanner.go to find it" — this is intentional. The scanner's write helpers are too deeply embedded for me to enumerate without reading more code; the implementer subagent will discover them. If a subagent reports BLOCKED on Task 8 because the write helpers aren't shaped how this plan assumes, the controller dispatches a discovery sub-task.
|
||||
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
**Goal:** Port the plugin's `podcastfeed` package as a first-party scheduled task in silo, so RSS-subscribed podcasts get their episode lists kept up to date.
|
||||
|
||||
**Source plugin:** `../silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
**Source plugin:** `/opt/silo_plugins/silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1: Port the refresher package
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `internal/audiobooks/podcastfeed/`
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `/opt/silo-server/internal/audiobooks/podcastfeed/`
|
||||
- Update imports (drop plugin SDK; use silo's `pgxpool` + `log/slog`)
|
||||
- Rewrite SQL queries against silo's `podcast_feeds` table (created in sub-plan 1 migration 140), `media_items` (type='podcast'), and `episodes`
|
||||
- Drop the plugin's `presentation_libraries` reference; subscriptions are tracked by `podcast_feeds.media_item_id`
|
||||
|
||||
@@ -979,7 +979,7 @@ Expected: `healthy`.
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT version_id FROM public.goose_db_version WHERE version_id=174 AND is_applied;"
|
||||
"SELECT version, filename FROM schema_versions WHERE version=174;"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='autoscan_sources' AND column_name='label';"
|
||||
```
|
||||
|
||||
@@ -1,617 +0,0 @@
|
||||
# TheIntroDB read-path correctness (Phase 1 / Layer A) — implementation plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Fix the three read-path correctness gaps in the TheIntroDB provider so it matches the
|
||||
real v3 API contract: (1) honor **TVDB** ids in lookups (today they are silently dropped),
|
||||
(2) capture and use the **real per-segment `confidence`** instead of a hardcoded `0.9`, and
|
||||
(3) when the API returns multiple candidate segments, **pick the most-submitted / highest-confidence**
|
||||
one instead of the first usable. This is Phase 1 of
|
||||
[the marker sources & contribution design](../specs/2026-06-06-marker-sources-and-contribution-design.md);
|
||||
it stands alone, ships value immediately (anime / TheTVDB-first libraries start getting markers),
|
||||
and unblocks the "query-all / best-wins" merge in later phases.
|
||||
|
||||
**Architecture:** All changes are contained within `internal/markers/introdb` plus one added
|
||||
field on `markers.Marker`. The resolver already populates the TVDB id into
|
||||
`Request.ExternalIDs` — only the provider and HTTP client drop it. The HTTP `Client` gains a
|
||||
`tvdbID` argument on `FetchEpisode`/`FetchMovie` (query preference `tmdb → tvdb → imdb`, cache
|
||||
keys extended); the provider reads `ExternalIDKeyTVDB`; `segmentTimestamps` gains `confidence`
|
||||
and `submission_count`; `pickMarker` uses real confidence (with a named default) and ranks
|
||||
candidates. The write path (`BuildUpdatePayload` / `UpsertMarkers`) is **not** touched here.
|
||||
|
||||
**Out of scope (deferred to Phase 2):** threading *per-segment* confidence through
|
||||
`MarkerUpdatePayload` and `UpsertMarkers`. That refactor only becomes observable once
|
||||
`Registry.FetchMerged` can combine segments from different providers; with a single provider
|
||||
today the existing "max confidence across segments" collapse is equivalent, and changing it now
|
||||
would break `TestBuildUpdatePayloadAggregatesConfidence` for no functional gain. It lands with
|
||||
the merge dispatch in Phase 2.
|
||||
|
||||
**Tech Stack:** Go (the `internal/markers/introdb` package is pure Go — no libvips/CGO),
|
||||
PostgreSQL is not involved (no migration), no frontend changes. Tests run in a throwaway Go
|
||||
container (the host has no Go toolchain); a named volume caches module downloads.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
`internal/markers` and `internal/markers/introdb` are pure Go, so no libvips is needed:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
To run a single test, append `-run TestName` and narrow the package path, e.g.
|
||||
`go test ./internal/markers/introdb/ -run TestFetchEpisodeSendsTVDBWhenNoTMDB -v`.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
- `internal/markers/introdb/types.go` — **modify**. Add `Confidence` + `SubmissionCount` to `segmentTimestamps`; add the `defaultConfidence` const.
|
||||
- `internal/markers/types.go` — **modify**. Add `SubmissionCount` to `markers.Marker`.
|
||||
- `internal/markers/introdb/client.go` — **modify**. `tvdbID` arg on `FetchEpisode`/`FetchMovie`; query preference `tmdb → tvdb → imdb`; cache keys include tvdb.
|
||||
- `internal/markers/introdb/provider.go` — **modify**. Read `ExternalIDKeyTVDB` and pass it through; rewrite `pickMarker` for real confidence + best-candidate.
|
||||
- `internal/markers/introdb/client_test.go` — **create**. httptest-based client tests (TVDB query, TMDB preference, caching).
|
||||
- `internal/markers/introdb/provider_test.go` — **create**. End-to-end provider tests (TVDB-only resolves, real/default confidence, best-candidate).
|
||||
|
||||
No migration, no wiring change (`cmd/silo/main.go` constructs the provider but never calls the
|
||||
client directly), no frontend change.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Type plumbing — capture confidence + submission_count
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/types.go`
|
||||
- Modify: `internal/markers/types.go`
|
||||
|
||||
This task is pure struct plumbing (no behavior yet); later tasks' tests exercise it. Verify by
|
||||
compilation.
|
||||
|
||||
- [ ] **Step 1: Add the response fields + default constant in `introdb/types.go`**
|
||||
|
||||
Replace the `Algorithm` const block to add `defaultConfidence` after it:
|
||||
|
||||
```go
|
||||
// Algorithm is the algorithm tag written alongside markers. The version
|
||||
// suffix lets us invalidate or refresh markers if the upstream contract
|
||||
// changes.
|
||||
const Algorithm = "introdb:v3"
|
||||
|
||||
// defaultConfidence is applied when TheIntroDB omits a per-segment confidence
|
||||
// in the /media response. Real per-segment confidence is preferred when present.
|
||||
const defaultConfidence = 0.9
|
||||
```
|
||||
|
||||
Replace the `segmentTimestamps` struct with the version that decodes the two extra fields the
|
||||
v3 `/media` response carries:
|
||||
|
||||
```go
|
||||
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
|
||||
// Either bound may be nil — for intro/recap, start may be omitted (segment
|
||||
// begins at file start); for credits/preview, end may be omitted (segment
|
||||
// runs to file end). Confidence and SubmissionCount are optional per-segment
|
||||
// quality signals used to pick among multiple candidates.
|
||||
type segmentTimestamps struct {
|
||||
StartMs *int64 `json:"start_ms,omitempty"`
|
||||
EndMs *int64 `json:"end_ms,omitempty"`
|
||||
Confidence *float64 `json:"confidence,omitempty"`
|
||||
SubmissionCount *int `json:"submission_count,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add `SubmissionCount` to `markers.Marker`**
|
||||
|
||||
In `internal/markers/types.go`, replace the `Marker` struct:
|
||||
|
||||
```go
|
||||
type Marker struct {
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
SubmissionCount int
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify the packages still compile**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./internal/markers/...
|
||||
```
|
||||
|
||||
Expected: no output, exit 0. (Existing `write_test.go` / `types_test.go` use field-keyed
|
||||
literals, so the added field does not break them.)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/types.go internal/markers/types.go
|
||||
git commit -m "feat(markers): decode introdb per-segment confidence + submission_count"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: TVDB lookups (gap #1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/client.go`
|
||||
- Modify: `internal/markers/introdb/provider.go`
|
||||
- Create: `internal/markers/introdb/client_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing client tests**
|
||||
|
||||
Create `internal/markers/introdb/client_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchEpisodeSendsTVDBWhenNoTMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "55555", "tt1234567", 2, 3, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "55555" {
|
||||
t.Errorf("tvdb_id = %q, want 55555", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "" {
|
||||
t.Errorf("tmdb_id = %q, want empty", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("imdb_id should be omitted when tvdb present, got %q", gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodePrefersTMDBOverTVDBAndIMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "111", "222", "tt333", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "111" {
|
||||
t.Errorf("tmdb_id = %q, want 111", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "" || gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("only tmdb_id expected, got tvdb=%q imdb=%q", gotQuery.Get("tvdb_id"), gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchMovieSendsTVDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"movie"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchMovie(context.Background(), "", "888", "", 0); err != nil {
|
||||
t.Fatalf("FetchMovie: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "888" {
|
||||
t.Errorf("tvdb_id = %q, want 888", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodeCachesByID(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "999", "", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Fatalf("server hits = %d, want 1 (cached after first)", hits)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: FAIL — compile error (`FetchEpisode`/`FetchMovie` still take the old 6/4-arg
|
||||
signatures without `tvdbID`).
|
||||
|
||||
- [ ] **Step 3: Add `tvdbID` to `FetchEpisode` in `client.go`**
|
||||
|
||||
Replace the whole `FetchEpisode` method:
|
||||
|
||||
```go
|
||||
// FetchEpisode looks up segment timestamps for a TV episode.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
if season <= 0 || episode <= 0 {
|
||||
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
q.Set("season", strconv.Itoa(season))
|
||||
q.Set("episode", strconv.Itoa(episode))
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, tvdbID, imdbID, season, episode, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `tvdbID` to `FetchMovie` in `client.go`**
|
||||
|
||||
Replace the whole `FetchMovie` method:
|
||||
|
||||
```go
|
||||
// FetchMovie looks up segment timestamps for a movie.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchMovie(ctx context.Context, tmdbID, tvdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, tvdbID, imdbID, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Extend the cache-key helpers in `client.go`**
|
||||
|
||||
Replace both `cacheKeyEpisode` and `cacheKeyMovie`:
|
||||
|
||||
```go
|
||||
func cacheKeyEpisode(tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:%s:s%de%d:d%d", tvdbID, season, episode, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheKeyMovie(tmdbID, tvdbID, imdbID string, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:movie:%s:d%d", tvdbID, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Read TVDB in the provider and pass it through (`provider.go`)**
|
||||
|
||||
Replace the id-extraction block in `FetchMarkers` (the `tmdbID`/`imdbID` lines and the empty
|
||||
guard):
|
||||
|
||||
```go
|
||||
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
|
||||
tvdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTVDB])
|
||||
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
```
|
||||
|
||||
Then update the two client calls in the `switch req.Kind` block to pass `tvdbID`:
|
||||
|
||||
```go
|
||||
resp, err = p.client.FetchEpisode(ctx, tmdbID, tvdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
|
||||
```
|
||||
```go
|
||||
resp, err = p.client.FetchMovie(ctx, tmdbID, tvdbID, imdbID, durationMS)
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run the client tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestFetch*` tests).
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/client.go internal/markers/introdb/provider.go internal/markers/introdb/client_test.go
|
||||
git commit -m "fix(markers): honor TVDB ids in TheIntroDB lookups"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Real confidence + best-candidate selection (gaps #2, #3)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/provider.go` (`pickMarker`)
|
||||
- Create: `internal/markers/introdb/provider_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing provider tests**
|
||||
|
||||
Create `internal/markers/introdb/provider_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
func newProvider(t *testing.T, body string) *Provider {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
return NewProvider(c)
|
||||
}
|
||||
|
||||
func episodeReq(ids map[string]string) markers.Request {
|
||||
return markers.Request{
|
||||
Kind: markers.ItemKindEpisode,
|
||||
ExternalIDs: ids,
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Duration: 30 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderResolvesTVDBOnly(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, err := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTVDB: "777"}))
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMarkers: %v", err)
|
||||
}
|
||||
if len(res.Markers) != 1 || res.Markers[0].Kind != markers.MarkerKindIntro {
|
||||
t.Fatalf("expected one intro marker, got %+v", res.Markers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderUsesRealConfidence(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000,"confidence":0.42}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if res.Markers[0].Confidence != 0.42 {
|
||||
t.Errorf("confidence = %v, want 0.42 (real value, not hardcoded)", res.Markers[0].Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderDefaultsConfidenceWhenAbsent(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if res.Markers[0].Confidence != defaultConfidence {
|
||||
t.Errorf("confidence = %v, want default %v", res.Markers[0].Confidence, defaultConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPicksMostSubmittedCandidate(t *testing.T) {
|
||||
body := `{"type":"episode","intro":[
|
||||
{"end_ms":50000,"confidence":0.6,"submission_count":2},
|
||||
{"end_ms":61000,"confidence":0.5,"submission_count":9}
|
||||
]}`
|
||||
p := newProvider(t, body)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if got := res.Markers[0].End; got != 61*time.Second {
|
||||
t.Errorf("picked end = %v, want 61s (the submission_count=9 candidate)", got)
|
||||
}
|
||||
if res.Markers[0].SubmissionCount != 9 {
|
||||
t.Errorf("submission_count = %d, want 9", res.Markers[0].SubmissionCount)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: FAIL — `TestProviderUsesRealConfidence` sees `0.9` (hardcoded), `TestProviderPicksMostSubmittedCandidate`
|
||||
gets the first usable (`end=50s`, `SubmissionCount=0`). (`TestProviderResolvesTVDBOnly` already
|
||||
passes from Task 2 — it guards against regression.)
|
||||
|
||||
- [ ] **Step 3: Rewrite `pickMarker` in `provider.go`**
|
||||
|
||||
Replace the entire `pickMarker` function:
|
||||
|
||||
```go
|
||||
// pickMarker selects the best usable segment from a TheIntroDB response array.
|
||||
// `requireEnd` is true for segments where the end timestamp is the load-bearing
|
||||
// field (intro, recap) — they're allowed to start at 0 if `start_ms` is omitted.
|
||||
// For trailing segments (credits, preview) the start is required but the end
|
||||
// defaults to the file duration. When several candidates are usable (e.g. no
|
||||
// duration match narrowed the set), the most-submitted one wins, with higher
|
||||
// confidence breaking ties; with a single candidate this is the previous
|
||||
// first-usable behavior. Real per-segment confidence is used when present,
|
||||
// falling back to defaultConfidence only when the API omits it.
|
||||
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
|
||||
best := markers.Marker{}
|
||||
bestSubs := -1
|
||||
found := false
|
||||
for _, s := range stamps {
|
||||
start := time.Duration(0)
|
||||
end := totalDuration
|
||||
if s.StartMs != nil {
|
||||
start = time.Duration(*s.StartMs) * time.Millisecond
|
||||
}
|
||||
if s.EndMs != nil {
|
||||
end = time.Duration(*s.EndMs) * time.Millisecond
|
||||
}
|
||||
if requireEnd && s.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
if !requireEnd && s.StartMs == nil {
|
||||
continue
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
confidence := defaultConfidence
|
||||
if s.Confidence != nil {
|
||||
confidence = *s.Confidence
|
||||
}
|
||||
subs := 0
|
||||
if s.SubmissionCount != nil {
|
||||
subs = *s.SubmissionCount
|
||||
}
|
||||
if !found || subs > bestSubs || (subs == bestSubs && confidence > best.Confidence) {
|
||||
best = markers.Marker{Kind: kind, Start: start, End: end, Confidence: confidence, SubmissionCount: subs}
|
||||
bestSubs = subs
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestProvider*` tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/provider.go internal/markers/introdb/provider_test.go
|
||||
git commit -m "feat(markers): use real introdb confidence and pick best candidate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Full-package regression + integration build
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Run the entire markers test suite**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
Expected: PASS — the new `introdb` tests plus the existing `write_test.go` /
|
||||
`types_test.go` (unchanged behavior: `BuildUpdatePayload` still aggregates max confidence,
|
||||
`FetchFirstHit` still uses registration order).
|
||||
|
||||
- [ ] **Step 2: Build the binary to confirm nothing downstream broke**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./...
|
||||
```
|
||||
|
||||
Expected: exit 0. (The only callers of the changed client methods are in `provider.go`, updated
|
||||
in Task 2; `cmd/silo/main.go` constructs the provider but never calls the client directly.)
|
||||
|
||||
- [ ] **Step 3: Confirm the working tree is clean**
|
||||
|
||||
```bash
|
||||
git status
|
||||
```
|
||||
|
||||
Expected: clean (all changes committed in Tasks 1–3; this task adds no files).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Gap coverage:** TVDB lookups (Task 2, `TestFetch*` + `TestProviderResolvesTVDBOnly`), real
|
||||
confidence (Task 3, `TestProviderUsesRealConfidence` + `…DefaultsConfidenceWhenAbsent`),
|
||||
best-candidate (Task 3, `TestProviderPicksMostSubmittedCandidate`). Maps to design gaps #1–#3.
|
||||
- **Containment:** the client signature change touches exactly one production caller
|
||||
(`provider.go`); `markers.Marker` is constructed only in `provider.go`. Verified by grep before
|
||||
planning, re-verified by the Task 4 `go build ./...`.
|
||||
- **No behavior regressions:** the write path is untouched, so `BuildUpdatePayload`'s
|
||||
max-confidence aggregation and its test remain valid; per-segment confidence is deferred to
|
||||
Phase 2 where the merge actually consumes it.
|
||||
- **No placeholders:** every code step shows the complete replacement; every run step shows the
|
||||
command and the expected result. No migration, no frontend, no wiring changes.
|
||||
- **Default confidence:** `defaultConfidence = 0.9` preserves today's effective value for
|
||||
responses that omit confidence, so existing stored markers keep equivalent provenance.
|
||||
@@ -1,482 +0,0 @@
|
||||
# Marker sources & TheIntroDB contribution — implementation plan (all phases)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Implement phases in order; each phase is independently shippable.
|
||||
|
||||
**Goal:** Implement the design in
|
||||
[2026-06-06-marker-sources-and-contribution-design.md](../specs/2026-06-06-marker-sources-and-contribution-design.md):
|
||||
fix the TheIntroDB read-path correctness gaps, formalize a multi-source provider model
|
||||
(query-all / best-wins), and add the ability to contribute Silo's own markers back to
|
||||
TheIntroDB — configured **per provider, off by default**. Server-side API + data model only;
|
||||
web UI is a separate follow-up.
|
||||
|
||||
**Architecture:** Five sequential phases. Phase 1 fixes the existing provider (TVDB, real
|
||||
confidence, best-candidate). Phase 2 adds a per-provider config table, per-segment marker
|
||||
provenance, and `Registry.FetchMerged`. Phase 3 adds the submission client, the contribution
|
||||
audit table, and the `ContributionService` engine. Phase 4 exposes the full admin API (manual
|
||||
markers, provider config, contribute, history). Phase 5 adds the daily auto-contribution task.
|
||||
|
||||
**Tech Stack:** Go (chi handlers, pgx repositories), PostgreSQL (paired numbered migrations),
|
||||
no frontend in this plan. Most touched packages (`internal/markers`, `internal/markers/introdb`,
|
||||
`internal/taskmanager/tasks`) are pure Go; `internal/api/handlers` needs libvips (CGO image
|
||||
deps). Tests run in a throwaway Go container (host has no Go toolchain).
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Phase overview & dependencies
|
||||
|
||||
| Phase | Delivers | Depends on | New migrations |
|
||||
|------|----------|-----------|----------------|
|
||||
| 1 | TheIntroDB read-path correctness (TVDB, real confidence, best-candidate) | — | none |
|
||||
| 2 | `marker_provider_config` + per-segment provenance + `FetchMerged` | 1 (real confidence) | `marker_provider_config` |
|
||||
| 3 | Submission client + `marker_contributions` + `ContributionService` | 2 (`Submitter`, provider config) | `marker_contributions` |
|
||||
| 4 | Admin API: manual markers, provider config, contribute, history | 3 (service) | none |
|
||||
| 5 | Daily auto-contribution task | 3 (service), 4 (optional) | none |
|
||||
|
||||
Phase 1's line-level steps are also captured standalone in
|
||||
[2026-06-06-marker-introdb-readpath-correctness.md](2026-06-06-marker-introdb-readpath-correctness.md);
|
||||
they are summarized here so this document is self-contained.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
Pure-Go packages (`internal/markers`, `internal/markers/introdb`, `internal/taskmanager/...`):
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
`internal/api/handlers` (Phase 4) needs libvips:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
sh -c 'apt-get update >/dev/null && apt-get install -y --no-install-recommends libvips-dev >/dev/null && go test ./internal/api/handlers/ -run TestAdminMarkers -v'
|
||||
```
|
||||
|
||||
Whole-project compile: `go build ./...` (same container).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — TheIntroDB read-path correctness
|
||||
|
||||
**Goal:** Honor TVDB ids, capture/use real per-segment confidence + submission_count, and pick
|
||||
the best candidate among multiple. Contained to `internal/markers/introdb` plus one field on
|
||||
`markers.Marker`. No migration, no write-path change.
|
||||
|
||||
**Files:** `internal/markers/introdb/{types.go,client.go,provider.go}` (modify),
|
||||
`internal/markers/types.go` (modify, add `Marker.SubmissionCount`),
|
||||
`internal/markers/introdb/{client_test.go,provider_test.go}` (create).
|
||||
|
||||
### Task 1.1 — Type plumbing
|
||||
- [ ] Add `Confidence *float64` and `SubmissionCount *int` to `segmentTimestamps`
|
||||
(`introdb/types.go`); add `const defaultConfidence = 0.9`.
|
||||
- [ ] Add `SubmissionCount int` to `markers.Marker` (`markers/types.go`).
|
||||
- [ ] `go build ./internal/markers/...` → exit 0. Commit.
|
||||
|
||||
### Task 1.2 — TVDB lookups (gap #1)
|
||||
- [ ] Create `introdb/client_test.go` (httptest): asserts `tvdb_id` sent when only tvdb given,
|
||||
tmdb preferred over tvdb/imdb, movie tvdb, and cache collapses repeat calls. Run → FAIL (compile).
|
||||
- [ ] `client.go`: `FetchEpisode`/`FetchMovie` gain a `tvdbID` arg; query preference
|
||||
`tmdb → tvdb → imdb` via a `switch`; the empty-id guard includes tvdb; `cacheKeyEpisode`/
|
||||
`cacheKeyMovie` include a `tvdb:` branch.
|
||||
- [ ] `provider.go` `FetchMarkers`: read `req.ExternalIDs[markers.ExternalIDKeyTVDB]`, include it
|
||||
in the empty guard, pass it to both client calls.
|
||||
- [ ] Run client tests → PASS. Commit.
|
||||
|
||||
### Task 1.3 — Real confidence + best-candidate (gaps #2, #3)
|
||||
- [ ] Create `introdb/provider_test.go`: TVDB-only resolves; real confidence used; default when
|
||||
absent; most-submitted candidate wins (ties broken by confidence). Run → FAIL.
|
||||
- [ ] Rewrite `pickMarker` to use real confidence (fallback `defaultConfidence`), set
|
||||
`SubmissionCount`, and select the candidate with the highest `(submission_count, confidence)`.
|
||||
- [ ] Run → PASS. Commit.
|
||||
|
||||
### Task 1.4 — Regression
|
||||
- [ ] `go test ./internal/markers/... && go build ./...` → all green (existing `write_test.go`/
|
||||
`types_test.go` unaffected; write path untouched). Commit if needed.
|
||||
|
||||
(Full code for every step is in the standalone Phase 1 plan linked above.)
|
||||
|
||||
---
|
||||
|
||||
# Phase 2 — Per-provider config, per-segment provenance, FetchMerged
|
||||
|
||||
**Goal:** Add the `marker_provider_config` table that drives which providers are queried (and,
|
||||
later, submitted to); make the marker write path carry **per-segment** provider/confidence/
|
||||
algorithm so a merged multi-provider result writes correct provenance; add
|
||||
`Registry.FetchMerged` (query all enabled providers, keep the best candidate per segment) and
|
||||
switch the lazy-playback path to it. With only TheIntroDB enabled, behavior is unchanged.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/<next>_marker_provider_config.{up,down}.sql`
|
||||
- Create: `internal/markers/provider_config.go`, `internal/markers/provider_config_test.go`
|
||||
- Modify: `internal/markers/types.go` (`Marker.ProviderID`/`Algorithm`; `Registry.FetchMerged`)
|
||||
- Modify: `internal/markers/write.go` (per-segment provenance in `MarkerUpdatePayload`/`BuildUpdatePayload`)
|
||||
- Modify: `internal/scanner/file_repo.go` (`MarkerUpdate` + `applySegmentPatch` per-segment provenance)
|
||||
- Modify: the `markers.MarkerUpdatePayload` → `scanner.MarkerUpdate` conversion (in the lazy-markers handler/adapter)
|
||||
- Modify: `internal/api/handlers/playback_lazy_markers.go` (call `FetchMerged`)
|
||||
- Modify: `cmd/silo/main.go` (construct the config store; pass to the registry)
|
||||
|
||||
### Task 2.1 — Migration: `marker_provider_config`
|
||||
- [ ] Write `migrations/<next>_marker_provider_config.up.sql` (use the next free number; latest is 180):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY,
|
||||
fetch_enabled boolean NOT NULL DEFAULT true,
|
||||
fetch_priority integer NOT NULL DEFAULT 100,
|
||||
contribute_enabled boolean NOT NULL DEFAULT false,
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false,
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: `DROP TABLE public.marker_provider_config;`
|
||||
- [ ] Commit.
|
||||
|
||||
### Task 2.2 — Provider config store
|
||||
- [ ] Create `internal/markers/provider_config.go`:
|
||||
|
||||
```go
|
||||
type ProviderConfig struct {
|
||||
Provider string
|
||||
FetchEnabled bool
|
||||
FetchPriority int
|
||||
ContributeEnabled bool
|
||||
ContributeAutoLocal bool
|
||||
ContributeMinConfidence float64
|
||||
}
|
||||
|
||||
// ProviderConfigStore is a cached read/write façade over marker_provider_config.
|
||||
// Reads serve from an in-memory snapshot; Update writes through and refreshes.
|
||||
type ProviderConfigStore struct { /* pool, mu, cache map[string]ProviderConfig */ }
|
||||
|
||||
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore
|
||||
func (s *ProviderConfigStore) Reload(ctx context.Context) error // load all rows into cache
|
||||
func (s *ProviderConfigStore) List() []ProviderConfig // snapshot copy
|
||||
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool)
|
||||
func (s *ProviderConfigStore) Update(ctx context.Context, p ProviderConfig) error // upsert + refresh
|
||||
// EnabledForFetch returns fetch_enabled providers sorted by (fetch_priority asc, provider asc).
|
||||
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig
|
||||
```
|
||||
|
||||
- [ ] Hot-reload: subscribe the store's `Reload` to the existing `cache.EventSettingsChanged`
|
||||
Redis channel (mirror the `introdb.api_key` reload wiring in `cmd/silo/main.go`), or add a
|
||||
dedicated `marker_provider_config_changed` event published by `Update`. Either is acceptable;
|
||||
document which.
|
||||
- [ ] `provider_config_test.go`: with a real/pgxmock pool or an injected fake, assert
|
||||
`EnabledForFetch` filters disabled rows and sorts by priority. Commit.
|
||||
|
||||
### Task 2.3 — Per-segment marker provenance (write path)
|
||||
|
||||
This makes a merged result (intro from provider A, credits from provider B) persist the correct
|
||||
per-segment `*_markers_provider` / `*_markers_confidence` / `*_markers_algorithm`. The DB columns
|
||||
already exist; today `applySegmentPatch` applies one shared triple.
|
||||
|
||||
- [ ] Add to `markers.Marker`: `ProviderID string`, `Algorithm string` (so each segment carries
|
||||
its origin). `provider.go` (Phase 1) sets these from `ProviderID`/`Algorithm` per marker.
|
||||
- [ ] Rework `markers.MarkerUpdatePayload` (`write.go`) from one shared `Confidence`/`Provider`/
|
||||
`Algorithm` to a per-segment struct, e.g.:
|
||||
|
||||
```go
|
||||
type SegmentPayload struct { Start, End *float64; Provider *string; Confidence *float64; Algorithm string }
|
||||
type MarkerUpdatePayload struct {
|
||||
Intro, Credits, Recap, Preview SegmentPayload
|
||||
Source string // shared source class (e.g. "online"); per-kind provider differs
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Update `BuildUpdatePayload` to fill each `SegmentPayload` from the matching `Marker`
|
||||
(provider/confidence/algorithm per segment). Update `write_test.go`
|
||||
(`TestBuildUpdatePayloadAggregatesConfidence` → assert per-segment confidence: intro 0.7,
|
||||
credits 0.9, recap 0.5).
|
||||
- [ ] Read `applySegmentPatch` and the `MarkerUpdatePayload → scanner.MarkerUpdate` conversion
|
||||
first, then extend `scanner.MarkerUpdate` + `applySegmentPatch` so each segment's
|
||||
provider/confidence/algorithm come from that segment (not a shared field). `nextSharedMarkerAttribution`
|
||||
keeps populating the legacy shared `markers_source`/`markers_confidence` columns from the
|
||||
highest-confidence applied segment.
|
||||
- [ ] `go test ./internal/markers/... ./internal/scanner/...` → green. Commit.
|
||||
|
||||
### Task 2.4 — `Registry.FetchMerged`
|
||||
- [ ] Give `Registry` access to the enabled set (constructor takes a `ProviderConfigStore`, or
|
||||
`FetchMerged` takes an `[]ProviderConfig` snapshot). Add:
|
||||
|
||||
```go
|
||||
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
|
||||
// segment kind, the candidate with the highest (SubmissionCount, Confidence),
|
||||
// breaking ties by the provider's fetch_priority. Per-segment ProviderID/
|
||||
// Algorithm are preserved so the write path records correct provenance.
|
||||
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error)
|
||||
```
|
||||
|
||||
Algorithm: fan out via goroutines (bounded) to enabled providers; collect `(ProviderConfig,
|
||||
Result, error)`; for each `MarkerKind`, choose the best `Marker` across results; assemble a
|
||||
`Result` whose `Markers` carry their winning provider's `ProviderID`/`Algorithm`. Log and skip
|
||||
provider errors (like `FetchFirstHit`). Return `ok=false` if no segment found.
|
||||
|
||||
- [ ] `types_test.go`: add a merge test — two fake providers, one wins intro, the other wins
|
||||
credits by submission_count; assert the merged result has both with correct per-segment
|
||||
ProviderID. Keep the existing `FetchFirstHit` tests. Commit.
|
||||
|
||||
### Task 2.5 — Switch lazy path + wire store
|
||||
- [ ] In `internal/api/handlers/playback_lazy_markers.go`, replace the `FetchFirstHit` call in
|
||||
`fetchOnlineMarkersForPlayback` with `FetchMerged`.
|
||||
- [ ] In `cmd/silo/main.go`, construct `NewProviderConfigStore`, `Reload` it at startup, wire its
|
||||
hot-reload, and pass it to the registry. (TheIntroDB stays registered as today.)
|
||||
- [ ] `go build ./... && go test ./internal/markers/...` → green. With only `introdb` enabled,
|
||||
`FetchMerged` returns the same result `FetchFirstHit` did — verify a provider test asserting
|
||||
single-provider parity. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 3 — Submission client, contribution tracking, service engine
|
||||
|
||||
**Goal:** Build the machinery to submit a marker to a provider, idempotently and audited —
|
||||
without yet wiring a trigger. Adds the `Submitter` capability, the introdb submission/stats
|
||||
client methods, the `marker_contributions` table, and the `ContributionService`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/types.go` (`Submitter`, `SubmissionRequest`/`SubmissionResult`/`UserStats`)
|
||||
- Modify: `internal/markers/introdb/{types.go,client.go,provider.go}` (submit + stats; implement `Submitter`)
|
||||
- Create: `migrations/<next>_marker_contributions.{up,down}.sql`
|
||||
- Create: `internal/markers/contribution_repo.go`, `internal/markers/contribute.go` (+ tests)
|
||||
- Modify: `cmd/silo/main.go` (construct the service)
|
||||
|
||||
### Task 3.1 — `Submitter` capability + DTOs
|
||||
- [ ] Add to `internal/markers/types.go`:
|
||||
|
||||
```go
|
||||
type SubmissionRequest struct {
|
||||
Kind ItemKind
|
||||
ExternalIDs map[string]string // tmdb/imdb (tvdb not accepted by /submit; tmdb required)
|
||||
SeasonNumber int
|
||||
EpisodeNumber int
|
||||
Segment MarkerKind
|
||||
Start, End *time.Duration // nil start ok for intro/recap; nil end ok for credits/preview
|
||||
Duration time.Duration
|
||||
}
|
||||
type SubmissionResult struct { ID string; Status string; Weight float64 } // status: pending|accepted|rejected
|
||||
type UserStats struct { Total, Accepted, Pending, Rejected int; AcceptanceRate float64; CurrentStreak, BestStreak int }
|
||||
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Commit (compiles; no implementer yet).
|
||||
|
||||
### Task 3.2 — introdb submission client
|
||||
- [ ] `introdb/types.go`: add the request/response shapes for `POST /v3/submit`
|
||||
(`tmdb_id`,`imdb_id`,`type`,`segment`,`season`,`episode`,`video_duration_ms`,`start_ms`,`end_ms`;
|
||||
response `{submissions:[{id,status,weight,...}]}`) and `GET /v3/user/stats`. Remove the
|
||||
"Submissions are intentionally not supported" package note.
|
||||
- [ ] `introdb/client.go`: add `SubmitSegment(ctx, ...) (*submitResponse, error)` (POST,
|
||||
`Authorization: Bearer` **required** — return an error if `apiKey == ""`; honor
|
||||
`X-UsageLimit-Reset` on 429) and `FetchUserStats(ctx) (*userStats, error)` (GET).
|
||||
- [ ] `introdb/provider.go`: implement `markers.Submitter` — map `SubmissionRequest` to the
|
||||
client call (apply the v3 null rules: intro/recap drop a zero start to `null`; credits/preview
|
||||
send `null` end when absent), translate `SubmissionResult`/`UserStats`.
|
||||
- [ ] `client_test.go`/`provider_test.go`: httptest asserts the submit body shape, bearer header,
|
||||
and that an empty key errors before any HTTP call; stats parse. Commit.
|
||||
|
||||
### Task 3.3 — `marker_contributions` table + repo
|
||||
- [ ] Migration `<next>_marker_contributions.up.sql` (per design §C4):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL,
|
||||
segment_kind text NOT NULL,
|
||||
source text NOT NULL,
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL,
|
||||
submission_id uuid,
|
||||
status text NOT NULL,
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: drop the table.
|
||||
- [ ] `internal/markers/contribution_repo.go`: `ContributionStore` with `AlreadySubmitted(ctx,
|
||||
fileID, provider, kind, contentHash) (bool, error)`, `Record(ctx, ContributionRow) error`
|
||||
(upsert on the unique key), `ListByFile(ctx, fileID) ([]ContributionRow, error)`. Add a
|
||||
`ContentHash(kind string, startMs, endMs, durationMs *int64) string` helper (stable SHA-256).
|
||||
- [ ] Repo test (value-hash idempotency, list). Commit.
|
||||
|
||||
### Task 3.4 — `ContributionService`
|
||||
- [ ] `internal/markers/contribute.go`:
|
||||
|
||||
```go
|
||||
type ContributionService struct { /* registry, resolver, cfg *ProviderConfigStore, store *ContributionStore, log */ }
|
||||
|
||||
// ContributeFile submits the file's eligible markers to enabled submitter providers.
|
||||
// opts may scope to one provider and/or specific segment kinds. Auto callers pass
|
||||
// requireAutoLocal=true so only providers with contribute_auto_local participate.
|
||||
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error)
|
||||
```
|
||||
|
||||
Per provider (from `cfg.List()` filtered to `Submitter` + `contribute_enabled`, and
|
||||
`contribute_auto_local` when `opts.Auto`):
|
||||
1. **Eligibility (C2):** for each requested segment present on the file, require
|
||||
`source ∈ {scanner, manual}` (skip `online` — circular); resolve external IDs via the existing
|
||||
resolver (skip if none / tmdb required by `/submit`); for auto require `source=scanner`,
|
||||
`kind=intro`, `confidence ≥ contribute_min_confidence`.
|
||||
2. **Idempotency:** compute `ContentHash`; skip if `AlreadySubmitted`.
|
||||
3. **Submit:** `provider.(Submitter).SubmitMarker(...)`.
|
||||
4. **Persist:** `store.Record(...)` with returned status/submission_id (or `status="error"`).
|
||||
|
||||
- [ ] `contribute_test.go` with a fake `Submitter` + in-memory store: asserts online-sourced
|
||||
markers are skipped, sub-threshold auto markers skipped, duplicates skipped, a fresh manual
|
||||
marker submitted and recorded. Commit.
|
||||
|
||||
### Task 3.5 — Wire
|
||||
- [ ] `cmd/silo/main.go`: construct `ContributionService` (registry, resolver, config store,
|
||||
contribution store). Not yet called by any handler/task. `go build ./...` + tests green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 4 — Admin API (manual markers, provider config, contribute, history)
|
||||
|
||||
**Goal:** Expose the full backend contract under `RequireAdmin`. Mirror the structure of
|
||||
`internal/api/handlers/admin_intro.go` (interface-typed deps, `chi.URLParam`, `writeError`/
|
||||
`writeJSON`, `MarkerUpdateNotifier` after writes).
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/api/handlers/admin_markers.go` (+ test), `internal/api/handlers/admin_marker_providers.go` (+ test)
|
||||
- Modify: `internal/api/router.go` (register routes in the `RequireAdmin` group)
|
||||
- Modify: `cmd/silo/main.go` (construct + inject the handlers)
|
||||
|
||||
### Task 4.1 — Manual marker endpoints (design §C3)
|
||||
- [ ] `admin_markers.go`:
|
||||
- `GET /admin/files/{fileId}/markers` → current per-segment values + provenance
|
||||
(`source`,`provider`,`confidence`,`algorithm`,`detected_at`) for intro/recap/credits/preview.
|
||||
- `PUT /admin/files/{fileId}/markers` → body per segment `{start,end}` (seconds) or `null` to
|
||||
clear; write `source = "manual"` via `scanner.FileRepository.UpsertMarkers` (priority 4);
|
||||
then `MarkerUpdateNotifier.MarkersUpdated(file)`. Validation: `end>start`; intro/recap may
|
||||
omit start; credits/preview may omit end; within `[0, duration]`.
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` → clear one segment.
|
||||
- `GET|PUT /admin/items/{id}/markers` → resolve to the item's primary file (convenience).
|
||||
- [ ] Handler test (libvips container): PUT writes manual markers and returns provenance;
|
||||
invalid range → 400; clear nulls the segment. Commit.
|
||||
|
||||
### Task 4.2 — Provider config + validate endpoints (design §C7)
|
||||
- [ ] `admin_marker_providers.go`:
|
||||
- `GET /admin/markers/providers` → for each registered provider: id, `isSubmitter`, its
|
||||
`marker_provider_config` row, and (best-effort cached) `UserStats` for submitters with a key.
|
||||
- `PUT /admin/markers/providers/{provider}` → update the config row (validated; unknown → 404).
|
||||
- `POST /admin/markers/providers/{provider}/validate` → `Submitter.FetchUserStats`; non-submitter → 400.
|
||||
- [ ] Test: list returns introdb with its config; update flips `contribute_enabled`; validate on
|
||||
a non-submitter → 400. Commit.
|
||||
|
||||
### Task 4.3 — Contribution endpoints (design §C7)
|
||||
- [ ] In `admin_markers.go` (or a small `admin_contributions.go`):
|
||||
- `POST /admin/files/{fileId}/contribute` (optional body `{provider?, segments?[]}`) →
|
||||
`ContributionService.ContributeFile(...)`; returns per (provider, segment) outcome.
|
||||
- `GET /admin/files/{fileId}/contributions` → `ContributionStore.ListByFile`.
|
||||
- [ ] Test with a fake service/store. Commit.
|
||||
|
||||
### Task 4.4 — Routing + wiring
|
||||
- [ ] Register all new routes inside the existing `RequireAdmin` group in `internal/api/router.go`
|
||||
(mirror the `redetect-intro` registration). Inject the handlers in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/api/handlers/ -run 'TestAdminMarkers|TestAdminMarkerProviders'`
|
||||
(libvips container) → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 5 — Daily auto-contribution task
|
||||
|
||||
**Goal:** A scheduled task that, for each provider with `contribute_enabled &&
|
||||
contribute_auto_local`, submits high-confidence local intro detections that haven't been
|
||||
contributed. Mirror `internal/taskmanager/tasks/detect_intro_markers.go`.
|
||||
|
||||
**Files:** Create `internal/taskmanager/tasks/contribute_markers.go` (+ test); modify
|
||||
`cmd/silo/main.go` (register the task).
|
||||
|
||||
### Task 5.1 — `ContributeMarkersTask`
|
||||
- [ ] Implement the `taskmanager.Task` interface (`Key="contribute_markers"`, `Name`,
|
||||
`Description`, `Category=TaskCategoryLibrary`, `IsHidden() bool`, `DefaultTriggers()` daily e.g.
|
||||
`"04:00"` — after the 03:30 detection task, `Execute`).
|
||||
- [ ] `Execute`: short-circuit if no provider has `contribute_enabled && contribute_auto_local`
|
||||
(read `ProviderConfigStore`). Otherwise page through episode files with
|
||||
`intro_markers_source='scanner'` and `intro_markers_confidence ≥` the provider's
|
||||
`contribute_min_confidence` (a new `ContributionStore`/repo query, batched), and call
|
||||
`ContributionService.ContributeFile(file, ContributeOptions{Auto:true})`. Report progress; obey
|
||||
the introdb usage limit (the client already backs off on `X-UsageLimit-Reset`); idempotency
|
||||
makes interrupted runs resumable.
|
||||
- [ ] Test with fakes: enabled+auto submits eligible files; disabled → no-op; sub-threshold
|
||||
skipped. Commit.
|
||||
|
||||
### Task 5.2 — Register
|
||||
- [ ] Register the task where `DetectIntroMarkersTask` is registered in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/taskmanager/...` → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
## Final integration verification
|
||||
|
||||
**Files:** none (build + runtime).
|
||||
|
||||
- [ ] **Build the image** (compiles backend + frontend):
|
||||
```bash
|
||||
docker build --build-arg BUILD_REVISION=$(git rev-parse --short HEAD) --build-arg BUILD_DIRTY=false -t silo-server:markers-test .
|
||||
```
|
||||
- [ ] **Recreate the container** (applies the new migrations on startup) and wait for health:
|
||||
```bash
|
||||
docker compose up -d silo
|
||||
for i in $(seq 1 24); do s=$(docker inspect -f '{{.State.Health.Status}}' silo-silo-1); echo "$s"; [ "$s" = healthy ] && break; sleep 5; done
|
||||
```
|
||||
- [ ] **Verify migrations applied + tables exist:**
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT to_regclass('public.marker_provider_config'), to_regclass('public.marker_contributions');"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c "SELECT * FROM marker_provider_config;"
|
||||
```
|
||||
Expected: both tables present; an `introdb` row with `fetch_enabled=t, contribute_enabled=f`.
|
||||
- [ ] **Smoke the read path (TVDB):** trigger playback (or call the marker fetch) for a
|
||||
TVDB-only-matched episode and confirm markers now populate where they previously did not.
|
||||
- [ ] **Smoke contribution (manual, dry):** enable contribution for introdb against a **test/staging**
|
||||
TheIntroDB key, `PUT` a manual marker on a file, `POST /admin/files/{id}/contribute`, then
|
||||
`GET /admin/files/{id}/contributions` shows a `pending` row; a second contribute is idempotent
|
||||
(no new row). Use a non-production key so test data isn't submitted to the real database.
|
||||
- [ ] `git status` → clean (all changes committed per phase).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Phase independence:** each phase compiles, tests, and ships alone; with only `introdb`
|
||||
enabled and contribution off (defaults), Phases 2–5 are observationally inert until an operator
|
||||
enables a provider — so they can land ahead of any behavior change.
|
||||
- **Per-provider, off by default:** `contribute_enabled`/`contribute_auto_local` default `false`
|
||||
in the migration; the service refuses to submit otherwise; auto additionally gated on
|
||||
`contribute_auto_local`. The `provider` column on `marker_contributions` and the per-provider
|
||||
loop make multi-target submission first-class.
|
||||
- **Circular-contribution guard:** `source='online'` is never contributed (C2 eligibility) — has
|
||||
a dedicated unit test in Task 3.4.
|
||||
- **Idempotency:** value `content_hash` + the unique constraint mean repeat/interrupted runs
|
||||
don't double-submit, while a corrected value (new hash) does submit — matching TheIntroDB's
|
||||
weighted-average model.
|
||||
- **Write-path accuracy:** Task 2.3 requires reading `applySegmentPatch` and the
|
||||
`MarkerUpdatePayload → scanner.MarkerUpdate` conversion before editing; the per-segment columns
|
||||
already exist in `media_files`, so this is a plumbing change, not a schema change.
|
||||
- **Migration numbers:** `marker_provider_config` and `marker_contributions` take the next two
|
||||
free numbers at implementation time (181+; latest on disk is 180). Re-check for collisions
|
||||
before writing.
|
||||
- **Out of scope (tracked in the design doc):** all web UI; per-user TheIntroDB accounts; a
|
||||
`marker_provider.v1` plugin capability. Clean seams left for each.
|
||||
@@ -1,14 +1,13 @@
|
||||
# Audiobooks Absorption — Discovery Findings
|
||||
|
||||
Produced by sub-plan 1, Task 1. Locks data-model and integration
|
||||
decisions for the audiobook foundation migrations and downstream sub-plans.
|
||||
decisions for migrations 139–142 and downstream sub-plans.
|
||||
|
||||
## D1 — Next migration number
|
||||
|
||||
Original snapshot: `138_search_number_word_normalization.up.sql` was the
|
||||
highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
Highest existing migration: 138_search_number_word_normalization.up.sql
|
||||
Next available number: 139
|
||||
Sub-plan 1 will use migrations 139 through 142.
|
||||
|
||||
## D2 — `media_libraries` kind/type column
|
||||
|
||||
@@ -17,10 +16,10 @@ The spec refers to `media_libraries` but the actual table is `media_folders`.
|
||||
library content. Current values in production: `movies`, `series`, `mixed`.
|
||||
|
||||
Existing column 'type' (text) discriminates library content on `media_folders`.
|
||||
Task 4 (migration 159) should ADD the value `audiobooks` to the type vocabulary
|
||||
Task 4 (migration 141) should ADD the value `audiobooks` to the type vocabulary
|
||||
rather than add a new column. The audiobook scanner branch will set
|
||||
`media_folders.type = 'audiobooks'` for audiobook libraries. No schema change
|
||||
needed for the column itself; migration 159 becomes a no-op DDL migration that
|
||||
needed for the column itself; migration 141 becomes a no-op DDL migration that
|
||||
documents the new allowed value and adds any supporting indexes if needed.
|
||||
|
||||
No CHECK constraint or enum enforces the `type` column values, so adding
|
||||
|
||||
@@ -41,10 +41,9 @@ Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
### Out
|
||||
|
||||
- Audiobook requests flow (`silo-plugin-audiobook-requests` untouched).
|
||||
- Smart collections, share links, content restrictions, metadata-provider integration.
|
||||
- Smart collections, share links, content restrictions, custom metadata providers.
|
||||
- Embedding-powered "similar books" recommender.
|
||||
- External enrichment through the audiobook metadata plugin/provider — deferred;
|
||||
scanner extracts local tags and identifiers for v1.
|
||||
- External enrichment (OpenLibrary / Google Books) — deferred; file tags only for v1.
|
||||
- Group listening / cowatch parity for audiobooks (silo already has cowatch for
|
||||
video; revisit later if needed).
|
||||
|
||||
@@ -54,7 +53,7 @@ Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
|
||||
New top-level Go package: `internal/audiobooks/`.
|
||||
|
||||
```text
|
||||
```
|
||||
internal/audiobooks/
|
||||
abs/ ← ported from plugin's internal/abs/ (~6.8k LOC)
|
||||
ABS-compatible REST handlers, mounted under /abs/* and the
|
||||
@@ -91,9 +90,7 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
- ~7.4k LOC ported from the plugin.
|
||||
- ~1.5–2k LOC new TypeScript/TSX in silo's SPA.
|
||||
- ~400 LOC of new Go glue (scanner branches, router wiring, service.go).
|
||||
- Three new SQL migrations in the current stack (`147_abs_sessions`,
|
||||
`157_podcast_feeds`, `160_audiobooks_feature_flag`) plus one documented
|
||||
no-op migration for the already-present media-folder type column.
|
||||
- Two new SQL migrations (134, 135) and at most one column-add migration.
|
||||
|
||||
## Data model
|
||||
|
||||
@@ -114,9 +111,9 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### New tables (two migrations)
|
||||
|
||||
1. **`abs_sessions`** (migration 147) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token hash, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token_hash`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
1. **`abs_sessions`** (migration 134) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
|
||||
2. **`podcast_feeds`** (migration 157) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
2. **`podcast_feeds`** (migration 135) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
|
||||
### Schema-touch column add
|
||||
|
||||
@@ -127,8 +124,8 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### Plugin migrations NOT ported
|
||||
|
||||
Smart collections, share links, embeddings, content restrictions,
|
||||
metadata-provider integration, request provider, file cache, listening stats aggregates,
|
||||
Smart collections, share links, embeddings, content restrictions, custom
|
||||
metadata providers, request provider, file cache, listening stats aggregates,
|
||||
reading goals, notification prefs, standalone-mode tables, recommender
|
||||
embeddings. All represent features dropped from scope.
|
||||
|
||||
@@ -228,9 +225,8 @@ and directors. No new code, just two more role string constants.
|
||||
|
||||
### External enrichment
|
||||
|
||||
Deferred. The scanner extracts local tags and identity hints only. Canonical
|
||||
enrichment remains the responsibility of the audiobook metadata plugin/provider
|
||||
path and is not ported in this foundation pass.
|
||||
Deferred. The plugin's `enrich/` package (OpenLibrary / Google Books lookup)
|
||||
is not ported in v1. File tags only. Add later if needed.
|
||||
|
||||
### Scheduled task
|
||||
|
||||
|
||||
@@ -1,472 +0,0 @@
|
||||
# Marker sources & TheIntroDB contribution — design
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Status:** Draft (rev 2), pending review
|
||||
**Scope:** Review the current TheIntroDB marker integration, formalize a multi-source
|
||||
marker-provider model (query-all / best-wins), and add the ability to contribute Silo's own
|
||||
markers back to TheIntroDB. This round designs the **server-side API and data model only** —
|
||||
web-admin and mobile surfaces are deferred to a UI-only follow-up. No client (Android/Apple)
|
||||
changes required.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
**Rev 2 changes:** contribution is configured **per provider** (off by default); all new web
|
||||
UI is **deferred** and the backend API is specified in full instead.
|
||||
|
||||
## Decisions locked for this design
|
||||
|
||||
These were chosen up front and constrain everything below:
|
||||
|
||||
1. **Deliverable now:** design/plan only. No source changes land from this document; it is
|
||||
the blueprint a follow-up implementation plan expands.
|
||||
2. **What we contribute:** *both* local auto-detected markers *and* manual corrections. Local
|
||||
detection only produces **intro markers for episodes**, so credits/recap/preview and all
|
||||
movies can only be contributed via manual correction — this asymmetry drives the design.
|
||||
3. **Account model:** a single server-wide TheIntroDB account, reusing the existing
|
||||
`introdb.api_key` server setting. Per-user accounts are a noted future seam, not built now.
|
||||
4. **Multi-source dispatch:** query all enabled online sources and keep the highest-quality
|
||||
result per segment ("best wins"). This makes *real per-segment confidence* (gap #2 below)
|
||||
a hard prerequisite, not an optional nicety.
|
||||
5. **Contribution is configured per provider, off by default.** Every contribution-capable
|
||||
provider has its own enable switch and submit settings, so when more providers exist an
|
||||
operator chooses *which* ones to submit to. Both the master contribute switch and the
|
||||
auto-submit-local-detections switch default to off; a single setting flips each on.
|
||||
6. **API-first, UI deferred.** This round fully specifies the backend (manual marker
|
||||
read/write/clear, contribution trigger + status, per-provider provider config, key
|
||||
validation and stats) so any future surface — web admin or mobile — builds on a complete
|
||||
contract. No new web UI is designed here beyond noting where it will eventually attach.
|
||||
|
||||
## Problem
|
||||
|
||||
Silo currently detects intro/recap/credits/preview markers from two places — a local
|
||||
chromaprint/chapter analyzer (`internal/intromarkers`) and the online TheIntroDB provider
|
||||
(`internal/markers/introdb`) — reconciled by a source-priority ladder at write time. The
|
||||
online side is cleanly abstracted behind a `markers.Provider` interface and a
|
||||
`markers.Registry`, which is good, but three things are missing:
|
||||
|
||||
1. The current TheIntroDB provider has **correctness gaps** measured against the real v3 API
|
||||
contract (TVDB lookups dropped, real confidence discarded — see "Review findings").
|
||||
2. There is **no path to contribute** Silo's markers back to TheIntroDB, even though the v3
|
||||
API exposes a submission endpoint and the package was deliberately built read-only.
|
||||
3. The provider abstraction is **fetch-only and first-hit**, so it does not yet support
|
||||
"several online sources, keep the best answer" or "this source can also accept
|
||||
contributions, and I choose which sources to submit to."
|
||||
|
||||
## Review findings — current implementation
|
||||
|
||||
### What is already right (keep it)
|
||||
|
||||
- **Clean source abstraction.** `markers.Provider` (`ID()`, `FetchMarkers()`) + `markers.Registry`
|
||||
with `FetchFirstHit` (`internal/markers/types.go`). TheIntroDB is *not* hardcoded; a second
|
||||
provider is one `Register()` call (`cmd/silo/main.go`).
|
||||
- **Multi-source-aware schema.** `media_files` carries per-segment provenance for all four
|
||||
kinds: `{intro,credits,recap,preview}_markers_{source,provider,confidence,algorithm,detected_at}`
|
||||
(migrations 108/109/137).
|
||||
- **Write-time reconciliation across source classes.** `CanWriteMarker`
|
||||
(`internal/markers/write.go`) + `MarkerSourcePriority` (`internal/models/marker_source.go`):
|
||||
`manual(4) > online/plugin(3) > s3(2) > scanner(1)`, confidence breaks ties. Local detection
|
||||
uses its own priority-gated path (`intromarkers.Repository.PatchIntroMarker`).
|
||||
- **Solid HTTP hygiene** in `internal/markers/introdb/client.go`: conservative rate limiter
|
||||
(2 req/s, burst 5; documented limit is 30 req/10s), retry with exponential backoff,
|
||||
`Retry-After` handling, 24h TTL cache with negative caching, hot-reloadable API key over
|
||||
Redis pub/sub.
|
||||
|
||||
### Gaps (measured against TheIntroDB's own v3 client + Jellyfin plugin)
|
||||
|
||||
| # | Gap | Evidence | Impact |
|
||||
|---|-----|----------|--------|
|
||||
| 1 | **TVDB lookups silently dropped.** Provider reads only TMDB+IMDB; client can't send `tvdb_id`. | `internal/markers/introdb/provider.go` (reads `ExternalIDKeyTMDB`/`IMDB` only); `client.go` `FetchEpisode/FetchMovie` build `tmdb_id` else `imdb_id`. | Titles matched only by TVDB (very common for anime / TheTVDB-first libraries) get **zero** markers, even though the resolver already populates `TvdbID` and the v3 API + both official clients support `tvdb_id`. Correctness bug. |
|
||||
| 2 | **Real confidence + submission_count discarded; hardcoded 0.9.** | `internal/markers/introdb/types.go` `segmentTimestamps` has only `start_ms`/`end_ms`; `pickMarker` sets `Confidence: 0.9`. | The `/media` response carries per-segment `confidence` and `submission_count`. Without them, "best wins" merging can't rank sources, and write-time confidence tie-breaks are meaningless. Hard blocker for the chosen dispatch model. |
|
||||
| 3 | **Naive multi-candidate selection.** `pickMarker` takes the *first* usable entry. | `internal/markers/introdb/provider.go`. | When several candidates remain (e.g. no duration match), first-usable may not be the most-submitted/highest-confidence. Minor today (duration narrows the set), but compounds once we rank by confidence. |
|
||||
| 4 | **No contribution path** (by design). | `internal/markers/introdb/types.go` package doc: "Submissions are intentionally not supported." | The whole point of this effort. |
|
||||
| 5 | **Provider is fetch-only.** No submit capability in the interface. | `markers.Provider` in `internal/markers/types.go`. | Contribution needs an optional `Submitter` capability so non-contributing sources don't have to implement it, and so submission is per provider. |
|
||||
| 6 | **No key validation / contribution stats.** | `IntroDBCredentialCard` in `web/src/pages/admin-settings/IntegrationsSettings.tsx` only sets the key. | v3 exposes `GET /v3/user/stats` to validate a key and show contribution totals/streaks. Useful "connect & verify" probe, currently absent from the API. |
|
||||
| 7 | **Single collapsed confidence in the write payload.** `BuildUpdatePayload` promotes one max confidence across all segments. | `internal/markers/write.go`. | The DB has *per-segment* confidence columns, but a merged result (intro from source A, credits from source B) can't populate them independently until per-segment confidence is threaded through the payload. Coupled to gap #2. |
|
||||
|
||||
## Authoritative API contract (TheIntroDB v3)
|
||||
|
||||
Confirmed from TheIntroDB's own `theintrodb` npm client (`src/funcs.ts`, `src/types.ts`) and
|
||||
the official Jellyfin plugin (`TheIntroDbClient.cs`). Base URL: `https://api.theintrodb.org/v3`.
|
||||
|
||||
### `GET /media` — read (public; optional key)
|
||||
|
||||
Query: one of `tmdb_id` / `tvdb_id` / `imdb_id` (preference order tmdb → tvdb → imdb),
|
||||
plus `season` + `episode` together for TV, plus optional `duration_ms` (selects the closest
|
||||
release version). Optional `Authorization: Bearer <key>` additionally includes that user's
|
||||
*pending* submissions.
|
||||
|
||||
Response — each segment kind is an array of:
|
||||
```jsonc
|
||||
{ "start_ms": 30000|null, "end_ms": 90000|null, "confidence": 0.93|null, "submission_count": 7|null }
|
||||
```
|
||||
`start_ms:null` = "from the beginning" (intro/recap); `end_ms:null` = "to the end"
|
||||
(credits/preview). Arrays may hold multiple entries (multiple release versions) — pick by
|
||||
duration match, then by `submission_count`/`confidence`.
|
||||
|
||||
### `POST /submit` — contribute (requires key)
|
||||
|
||||
`Authorization: Bearer <user key>` required; submissions are credited to that account.
|
||||
Body (one segment per call):
|
||||
```jsonc
|
||||
{
|
||||
"tmdb_id": 1234, // required
|
||||
"imdb_id": "tt0903747", // optional
|
||||
"type": "tv" | "movie",
|
||||
"segment": "intro" | "recap" | "credits" | "preview",
|
||||
"season": 1, // TV only (number, or "1,2,3" multi-select)
|
||||
"episode": 1, // TV only
|
||||
"video_duration_ms": 2760000, // optional, strongly recommended (release-version match)
|
||||
"start_ms": 0|null, // intro/recap may be null (= beginning)
|
||||
"end_ms": 90000|null // credits/preview may be null (= end)
|
||||
}
|
||||
```
|
||||
Response: `{ "submissions": [ { "id": uuid, "status": "pending"|"accepted"|"rejected",
|
||||
"weight": number, ... } ] }`. Submissions enter as `pending`; the backend weight-averages
|
||||
timestamps across submissions and moderates to accepted/rejected.
|
||||
|
||||
Validation rules to mirror client-side: intro/recap require an **end**; credits/preview
|
||||
require a **start**; TV requires season+episode; movies omit them; max timestamp 6h
|
||||
(21,600,000 ms); `video_duration_ms` is 0 (unknown) or ≥ 300,000 ms.
|
||||
|
||||
### `GET /user/stats` — validate key + contribution stats
|
||||
|
||||
`Authorization: Bearer <key>`. Returns `{ total, accepted, pending, rejected,
|
||||
acceptance_rate, current_streak, best_streak, total_time_saved_ms, top_media }`. A 2xx with no
|
||||
`error` means the key is valid. This is both our key-validation probe and the stats display.
|
||||
|
||||
### Rate / usage limits
|
||||
|
||||
429 responses carry `X-RateLimit-{Limit,Remaining,Reset}` and `X-UsageLimit-{Limit,Remaining,Reset}`.
|
||||
Reads count against the rate limit; **submissions count against the usage limit**. The
|
||||
contribution pipeline must honor `X-UsageLimit-Reset` and back off — reuse the existing
|
||||
limiter and add usage-limit awareness.
|
||||
|
||||
## Target architecture
|
||||
|
||||
Three layers, each independently shippable. Layer A is pure correctness and unblocks B and C.
|
||||
|
||||
### Layer A — Correctness foundation (gaps #1, #2, #3, #7)
|
||||
|
||||
- **TVDB in the client + provider.** Add `tvdb_id` to the query builder in
|
||||
`internal/markers/introdb/client.go` (`FetchEpisode`/`FetchMovie` gain a `tvdbID` arg,
|
||||
preference tmdb → tvdb → imdb, cache keys extended). Provider reads
|
||||
`req.ExternalIDs[markers.ExternalIDKeyTVDB]`. Resolver already supplies it — no resolver
|
||||
change.
|
||||
- **Capture confidence + submission_count.** Extend `segmentTimestamps` with
|
||||
`Confidence *float64` and `SubmissionCount *int`; add `SubmissionCount int` to
|
||||
`markers.Marker`; `pickMarker` uses the real confidence (fallback to a named default
|
||||
constant, e.g. `0.9`, only when the field is null) instead of hardcoding.
|
||||
- **Best-candidate selection.** `pickMarker` ranks candidates by `(submission_count desc,
|
||||
confidence desc)` after duration filtering, instead of first-usable.
|
||||
- **Per-segment confidence in the payload.** Replace the single `Confidence *float64` on
|
||||
`markers.MarkerUpdatePayload` with per-segment confidence (e.g. `IntroConfidence`,
|
||||
`CreditsConfidence`, …) so a merged multi-source result writes the correct
|
||||
`{kind}_markers_confidence` per kind. Update `BuildUpdatePayload` and the
|
||||
`scanner.FileRepository.UpsertMarkers` column mapping accordingly.
|
||||
|
||||
### Layer B — Multi-source dispatch ("query all, best wins")
|
||||
|
||||
- **`Submitter` capability (optional interface).**
|
||||
```go
|
||||
// internal/markers/types.go
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
// FetchUserStats validates the configured key and returns contribution stats.
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
Only `introdb.Provider` implements it; the contribution service type-asserts
|
||||
`provider.(markers.Submitter)`. This is also what makes contribution per-provider: the
|
||||
service only ever submits to registered providers that satisfy `Submitter` *and* are enabled
|
||||
in `marker_provider_config` (C6).
|
||||
|
||||
- **Merge dispatch.** Add `Registry.FetchMerged(ctx, req) (Result, bool, error)` that fans out
|
||||
to all *fetch-enabled* providers concurrently (bounded), then merges **per segment kind**
|
||||
keeping the candidate with the highest `(submission_count, confidence)`. `FetchFirstHit`
|
||||
stays as the trivial single-provider path / fallback. With only TheIntroDB enabled,
|
||||
`FetchMerged` behaves identically to today — zero behavior change until a second source
|
||||
exists.
|
||||
|
||||
- **Cross-source-class reconciliation is unchanged.** `FetchMerged` reconciles *within* the
|
||||
online class; `CanWriteMarker` still arbitrates online vs scanner vs manual at write time.
|
||||
No double logic.
|
||||
|
||||
- **Provider enable/order config.** Driven by the `marker_provider_config` table (C6):
|
||||
`fetch_enabled` selects which providers are queried, `fetch_priority` is the merge
|
||||
tiebreaker. The metadata `library_provider_chains` DB pattern (`internal/metadata/chain.go`)
|
||||
is the upgrade path *if* per-library control is ever needed — noted, not built.
|
||||
|
||||
- **Future plugin sources (deferred seam).** A `marker_provider.v1` plugin capability would let
|
||||
third-party sources plug in. The clean seam is a `PluginMarkerProvider` adapter implementing
|
||||
`markers.Provider` (mirroring `internal/metadata/plugin_provider.go`), so the registry never
|
||||
changes. Cross-repo (silo-plugin-sdk) — explicitly out of scope here.
|
||||
|
||||
### Layer C — Contribution to TheIntroDB
|
||||
|
||||
#### C1. Submission client
|
||||
|
||||
Add to `internal/markers/introdb/client.go`:
|
||||
- `SubmitSegment(ctx, SubmissionRequest) (SubmissionResult, error)` → `POST /v3/submit`,
|
||||
Bearer key **required** (error early if `introdb.api_key` is empty), usage-limit aware.
|
||||
- `FetchUserStats(ctx) (UserStats, error)` → `GET /v3/user/stats`.
|
||||
`introdb.Provider` implements `markers.Submitter` by delegating to these.
|
||||
|
||||
#### C2. What is eligible to contribute
|
||||
|
||||
A marker is contributable iff:
|
||||
- the target provider is a registered `Submitter` with `contribute_enabled = true` (C6), and
|
||||
- its segment source ∈ `{scanner, manual}` (**never** `online` — that came *from* an online
|
||||
provider; contributing it back is circular), and
|
||||
- the file resolves to a usable external ID (tmdb/tvdb/imdb) via the existing resolver, and
|
||||
- for **auto** contribution: provider `contribute_auto_local = true`, source = `scanner`,
|
||||
kind = `intro`, confidence ≥ that provider's `contribute_min_confidence`. (Local detection is
|
||||
intro-only/episodes-only.)
|
||||
- for **manual** contribution: any kind, episodes or movies, source = `manual`.
|
||||
|
||||
#### C3. Manual marker API (write path; UI deferred)
|
||||
|
||||
Markers live on `media_files`, and an item can have several versions, so the manual API is
|
||||
**file-scoped** for precision, with an item-level convenience resolver. All under the existing
|
||||
`RequireAdmin` group (`internal/api/middleware/auth.go`). New `internal/api/handlers/admin_markers.go`:
|
||||
|
||||
- `GET /admin/files/{fileId}/markers` — current per-segment values **plus full provenance**
|
||||
(`source`, `provider`, `confidence`, `algorithm`, `detected_at`) for each of
|
||||
intro/recap/credits/preview. The read side an editor (or a script) needs.
|
||||
- `PUT /admin/files/{fileId}/markers` — upsert the **manual** layer. Body carries, per segment,
|
||||
either `{start, end}` (seconds) or `null` to clear that segment's manual value. Writes
|
||||
`source = manual` (priority 4, beats every other source) through the existing priority-gated
|
||||
`scanner.FileRepository.UpsertMarkers` path, then notifies live sessions via
|
||||
`playback.MarkerUpdateNotifier.MarkersUpdated` (mirrors `admin_intro.go`).
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` — explicit single-segment clear.
|
||||
- `GET|PUT /admin/items/{id}/markers` — item-level convenience that resolves to the primary
|
||||
file (and reports the per-version set), for callers that don't track file IDs.
|
||||
|
||||
**Clear semantics.** Clearing a manual segment nulls that segment's value and provenance;
|
||||
`markers_source` drops to the next-highest segment still present. A subsequent local
|
||||
re-detection or online lazy-fetch can repopulate it. (We can't "un-overwrite" a value a manual
|
||||
edit replaced, so clearing means "remove the manual value," not "restore the previous source.")
|
||||
|
||||
**Validation** mirrors the contribution rules so a manual edit can always be contributed:
|
||||
`end > start`; intro/recap may omit start (= 0); credits/preview may omit end (= file
|
||||
duration); values within [0, file duration].
|
||||
|
||||
A web editor (player-chrome control with a "use current time" helper) attaches to this API
|
||||
later; its UX is intentionally **out of scope** for this round.
|
||||
|
||||
#### C4. Contribution tracking (idempotency + audit)
|
||||
|
||||
The audit log that makes submission idempotent and lets us report status. Per-provider by
|
||||
construction (the `provider` column), so the same marker can be tracked/submitted to several
|
||||
providers independently.
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_contributions.up.sql (next free number at implementation time)
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL, -- 'introdb'
|
||||
segment_kind text NOT NULL, -- intro|recap|credits|preview
|
||||
source text NOT NULL, -- what we contributed: 'scanner'|'manual'
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL, -- hash(segment_kind,start_ms,end_ms,duration_ms)
|
||||
submission_id uuid, -- returned by /submit
|
||||
status text NOT NULL, -- pending|accepted|rejected|error
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
`content_hash` over the *value* gives idempotency: the same times are never resubmitted to the
|
||||
same provider, but a manual correction (different times → different hash) submits the new
|
||||
value — which is exactly how TheIntroDB's weighted averaging expects corrections to arrive.
|
||||
|
||||
#### C5. Contribution service + triggers
|
||||
|
||||
A single `internal/markers/contribute.go` `ContributionService` funnels both triggers:
|
||||
`resolve enabled submitter providers (C6) → per provider: eligibility (C2) → idempotency check
|
||||
(hash vs marker_contributions) → Submitter.SubmitMarker → persist result`. Triggers:
|
||||
- **Manual / on-demand:** `POST /admin/files/{fileId}/contribute` (C7) submits a file's eligible
|
||||
markers to enabled providers (optionally scoped to one provider/segments).
|
||||
- **Auto local detections:** a daily `ContributeMarkersTask` (`internal/taskmanager/tasks/`)
|
||||
that, for each provider with `contribute_enabled && contribute_auto_local`, scans for episode
|
||||
files with `intro_markers_source = 'scanner'` and `intro_markers_confidence ≥` that provider's
|
||||
`contribute_min_confidence` lacking a current-hash `marker_contributions` row, and submits
|
||||
them rate/usage-limited. Runs after the existing 03:30 detection task.
|
||||
|
||||
#### C6. Per-provider configuration
|
||||
|
||||
Contribution and fetch behavior is **per provider**, so a single global toggle won't do.
|
||||
Credentials stay in `server_settings` (sensitive handling: `introdb.api_key`); behavioral
|
||||
config moves to a new table keyed by provider id:
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_provider_config.up.sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY, -- 'introdb'
|
||||
fetch_enabled boolean NOT NULL DEFAULT true, -- queried during reads (Layer B)
|
||||
fetch_priority integer NOT NULL DEFAULT 100, -- merge tiebreaker
|
||||
contribute_enabled boolean NOT NULL DEFAULT false, -- master submit switch (off)
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false, -- auto-submit local detections (off)
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- A provider's submissions require `contribute_enabled = true`. With it on, **manual**
|
||||
corrections submit on demand; **auto** submission of local detections additionally requires
|
||||
`contribute_auto_local = true`. Both contribute flags default off — "easily enabled with a
|
||||
setting," exactly one row per provider.
|
||||
- This same table carries the Layer B fetch enable/order, so "which providers do we read from"
|
||||
and "which do we submit to" live in one place. New providers insert a row (contribute off).
|
||||
- Surfaced and edited through the provider API (C7), so the future admin surface lists
|
||||
providers generically rather than hard-coding `introdb`.
|
||||
|
||||
The `markers.mode` / `markers.lazy_playback` globals and the `introdb.api_key` credential are
|
||||
unchanged.
|
||||
|
||||
#### C7. Admin provider + contribution API (fully fledged)
|
||||
|
||||
The complete backend surface, all under `RequireAdmin`, so a UI is purely additive later:
|
||||
|
||||
- `GET /admin/markers/providers` — list every registered provider: id, whether it is a
|
||||
`Submitter`, its `marker_provider_config` row, and (best-effort, cached) live `/user/stats`
|
||||
for submitter providers that have a key.
|
||||
- `PUT /admin/markers/providers/{provider}` — update that provider's config row
|
||||
(`fetch_enabled`, `fetch_priority`, `contribute_enabled`, `contribute_auto_local`,
|
||||
`contribute_min_confidence`). Validated; unknown provider → 404.
|
||||
- `POST /admin/markers/providers/{provider}/validate` — validate the configured key and return
|
||||
fresh `UserStats` (calls `Submitter.FetchUserStats`). Non-submitter → 400.
|
||||
- `POST /admin/files/{fileId}/contribute` — submit this file's eligible markers (C2) to enabled
|
||||
submitter providers; optional body `{provider?, segments?[]}` to scope to one provider /
|
||||
specific kinds. Returns per (provider, segment) `SubmissionResult`. Honors idempotency (C4):
|
||||
unchanged values are skipped, not resubmitted.
|
||||
- `GET /admin/files/{fileId}/contributions` — contribution history for the file from
|
||||
`marker_contributions` (per provider/segment: status, submitted_at, error).
|
||||
|
||||
Plus the manual marker endpoints from C3. Together these cover read, edit, submit, and audit
|
||||
without any web code.
|
||||
|
||||
## Data model summary
|
||||
|
||||
- **New table `marker_contributions`** (C4) — idempotent submission audit, keyed per
|
||||
(file, provider, segment, value-hash).
|
||||
- **New table `marker_provider_config`** (C6) — per-provider fetch + contribute behavior,
|
||||
seeded with `introdb` (fetch on, contribute off).
|
||||
- **No new flat server settings.** `introdb.api_key`, `markers.mode`, `markers.lazy_playback`
|
||||
are unchanged; behavioral provider config lives in `marker_provider_config`, not string keys.
|
||||
- **No new marker columns.** Per-segment confidence columns already exist; Layer A only changes
|
||||
how they're *populated* (per-segment instead of collapsed). `submission_count` stays
|
||||
transient (merge-ranking only) unless a later need appears.
|
||||
|
||||
## Client / cross-repo impact
|
||||
|
||||
- **No Android/Apple changes required.** This adds no new marker *kinds* and no new
|
||||
client-facing read contract: markers still flow to clients via the existing `WatchDetail`
|
||||
fields, the jellycompat `GET /MediaSegments` endpoint, and the `markers_updated` realtime
|
||||
event. Manual editing and contribution are server-side (admin API); their eventual UI is
|
||||
deferred web-admin work.
|
||||
- **Plugin SDK:** untouched. `marker_provider.v1` is a deferred future capability (Layer B
|
||||
seam), not part of this work.
|
||||
- A future *per-user* contribution model (each user links their own TheIntroDB account) *would*
|
||||
be client work (a "Connect TheIntroDB" screen) — explicitly deferred.
|
||||
|
||||
## Build sequence
|
||||
|
||||
Each phase is shippable on its own; later phases depend only on earlier ones. Every phase is
|
||||
**backend-only** — no web UI.
|
||||
|
||||
1. **Phase 1 — Correctness foundation (Layer A).** TVDB lookups; capture
|
||||
confidence/submission_count; best-candidate selection; per-segment confidence in the
|
||||
payload. *Directly answers "are we doing this properly," low risk, no new surface.*
|
||||
2. **Phase 2 — Multi-source dispatch + provider config (Layer B + C6).** `Submitter` interface;
|
||||
`FetchMerged` query-all/best-wins; `marker_provider_config` table + repo driving fetch
|
||||
enable/order. *No behavior change with only TheIntroDB enabled.*
|
||||
3. **Phase 3 — Submission client + tracking + service (C1, C4, C5 core).** `SubmitSegment` /
|
||||
`FetchUserStats`; `marker_contributions` migration + repo; `ContributionService` (eligibility,
|
||||
idempotency, per-provider gating).
|
||||
4. **Phase 4 — Manual marker API (C3) + provider/contribution API (C7).** File- and item-level
|
||||
manual read/write/clear; provider list/config/validate; per-file contribute + history. The
|
||||
full admin contract, no UI.
|
||||
5. **Phase 5 — Auto-contribution (C5 task).** `ContributeMarkersTask`: per provider, when
|
||||
`contribute_enabled && contribute_auto_local`, submit episode intro markers above the
|
||||
provider's `contribute_min_confidence`, usage-limit aware, idempotent.
|
||||
|
||||
Deferred (separate UI-only follow-up): web admin surfaces over C3/C6/C7 — a marker editor in
|
||||
the player chrome and a provider/contribution panel in admin settings.
|
||||
|
||||
## Risks & trade-offs
|
||||
|
||||
- **Data-quality / pollution.** Auto-submitting chromaprint detections (confidence 0.65–0.90)
|
||||
risks polluting a provider's data. Mitigation: `contribute_enabled` and `contribute_auto_local`
|
||||
default **off** per provider, and the default `contribute_min_confidence` of 0.95 admits only
|
||||
chapter / chapter+silence detections even once enabled. Manual, high-trust corrections are the
|
||||
intended first contribution. Revisit chromaprint auto-submission after watching acceptance
|
||||
rates via `/user/stats`.
|
||||
- **Single-account weighting.** All contributions credit one server account; TheIntroDB's
|
||||
weighting may treat a single account's mass submissions differently than many users'. Note
|
||||
the per-user seam; acceptable for v1.
|
||||
- **Circular contribution.** Never contribute `source = online`. Enforced in C2 eligibility and
|
||||
worth a unit test.
|
||||
- **Usage limits.** Bulk auto-submission can hit the submission usage limit; the task must obey
|
||||
`X-UsageLimit-Reset` and resume next run. Idempotency means an interrupted run safely resumes.
|
||||
- **"Query all" cost.** `FetchMerged` issues one lookup per fetch-enabled provider; with only
|
||||
TheIntroDB it's identical to today, and per-provider caching bounds the cost as sources are
|
||||
added.
|
||||
- **Movies & non-intro segments** have no local detector, so their only contribution route is
|
||||
manual correction — acceptable and explicit.
|
||||
|
||||
## Open questions for review
|
||||
|
||||
1. **Contribute to all enabled providers, or require explicit per-call selection?** The design
|
||||
defaults `POST /admin/files/{id}/contribute` to "every enabled submitter," with an optional
|
||||
`provider` scope. Acceptable, or should submission always name its target provider?
|
||||
2. **Manual-clear behavior.** Clearing a manual segment removes the manual value and lets a
|
||||
later detection/fetch repopulate it (we can't restore the exact pre-edit value). Is
|
||||
"remove, then re-detect" the right contract, or do you want a marker history/undo stack?
|
||||
3. **Per-provider config home.** This design uses a `marker_provider_config` table (clean for N
|
||||
providers, future per-library override) rather than flat `server_settings` keys. Confirm the
|
||||
table is the direction you want.
|
||||
|
||||
## File-level blueprint
|
||||
|
||||
**Backend (Go) — Layer A**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `tvdb_id` query support; cache-key extension.
|
||||
- `internal/markers/introdb/types.go` — *modify*: `confidence`/`submission_count` fields.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: read TVDB; real confidence; best-candidate pick.
|
||||
- `internal/markers/types.go` — *modify*: per-segment confidence in `Marker`/Result plumbing.
|
||||
- `internal/markers/write.go` — *modify*: per-segment confidence in `MarkerUpdatePayload` + `BuildUpdatePayload`.
|
||||
- `internal/scanner/file_repo.go` — *modify*: map per-segment confidence in `UpsertMarkers`.
|
||||
|
||||
**Backend (Go) — Layer B + per-provider config**
|
||||
- `internal/markers/types.go` — *modify*: `Submitter` interface; `SubmissionRequest`/`Result`, `UserStats`; `Registry.FetchMerged`.
|
||||
- `internal/markers/provider_config.go` — *create*: `marker_provider_config` repo (load/update; drives fetch enable/order + contribute gating).
|
||||
- `migrations/<next>_marker_provider_config.{up,down}.sql` — *create*.
|
||||
|
||||
**Backend (Go) — Layer C**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `SubmitSegment` (`POST /v3/submit`), `FetchUserStats` (`GET /v3/user/stats`), usage-limit handling.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: implement `markers.Submitter`; drop the "submissions not supported" note in `types.go`.
|
||||
- `internal/markers/contribute.go` — *create*: `ContributionService` (eligibility, idempotency, per-provider gating, submit, persist).
|
||||
- `internal/markers/contribution_repo.go` — *create*: `marker_contributions` CRUD + value-hash check.
|
||||
- `migrations/<next>_marker_contributions.{up,down}.sql` — *create*.
|
||||
- `internal/api/handlers/admin_markers.go` — *create*: manual marker read/write/clear (C3) + per-file contribute + history (C7 file endpoints).
|
||||
- `internal/api/handlers/admin_marker_providers.go` — *create*: provider list/config/validate (C7 provider endpoints).
|
||||
- `internal/api/router.go` — *modify*: register the new `RequireAdmin` routes.
|
||||
- `internal/taskmanager/tasks/contribute_markers.go` — *create*: daily auto-contribution task.
|
||||
- `cmd/silo/main.go` — *modify*: wire `ContributionService`, provider-config repo, the task, and the `Submitter`.
|
||||
|
||||
**Frontend (React/TS) — deferred (not this round)**
|
||||
- Marker editor over `PUT /admin/files/{id}/markers`; provider/contribution panel over the C7
|
||||
endpoints; `IntroDBCredentialCard` "validate key & show stats". Listed for traceability only.
|
||||
|
||||
**Reference (read-only, not in this repo)**
|
||||
- TheIntroDB v3 contract verified against `github.com/TheIntroDB/theintrodb-npm`
|
||||
(`src/funcs.ts`, `src/types.ts`, `docs/`) and `github.com/TheIntroDB/jellyfin-plugin`
|
||||
(`TheIntroDB/Api/*.cs`).
|
||||
@@ -29,28 +29,24 @@ require (
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
github.com/pressly/goose/v3 v3.27.1
|
||||
github.com/zishang520/socket.io/v2 v2.5.0
|
||||
go.n16f.net/thumbhash v1.1.0
|
||||
golang.org/x/image v0.39.0
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -59,7 +55,6 @@ require (
|
||||
github.com/quic-go/quic-go v0.53.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||
@@ -69,14 +64,14 @@ require (
|
||||
github.com/zishang520/webtransport-go v0.9.1 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
@@ -95,14 +90,15 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/sony/sonyflake v1.3.0
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/text v0.36.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
@@ -2,12 +2,12 @@ entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0 h1:0yJpD0RP1AetGsWVCXoPih2ggLjN8hVp8blS+2RkIWU=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220 h1:tiwfULdh9yOW7HO6fYf8v+lXIsrCrGfp3GM8ymhrBME=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
@@ -44,7 +44,6 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
|
||||
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -52,8 +51,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
|
||||
@@ -111,8 +108,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
@@ -128,12 +125,10 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
@@ -145,8 +140,6 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
|
||||
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
@@ -156,32 +149,26 @@ github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2
|
||||
github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.27.1 h1:6uEvcprBybDmW4hcz3gYujhARhye+GoWKhEWyzD5sh4=
|
||||
github.com/pressly/goose/v3 v3.27.1/go.mod h1:maruOxsPnIG2yHHyo8UqKWXYKFcH7Q76csUV7+7KYoM=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.53.0 h1:QHX46sISpG2S03dPeZBgVIZp8dGagIaiu2FiVYvpCZI=
|
||||
github.com/quic-go/quic-go v0.53.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/sony/sonyflake v1.3.0 h1:tiB4Dlp0lnmKp/h6BLXA14P8Qi+LYS9+0QRpcrKHvg4=
|
||||
github.com/sony/sonyflake v1.3.0/go.mod h1:LORtCywH/cq10ZbyfhKrHYgAUGH7mOBa76enV9txy/Y=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -228,24 +215,22 @@ go.n16f.net/thumbhash v1.1.0 h1:aBEvuAd4yiwzeQ7Sm4BZoHJYbrQ1ewjrmrRlCE79snk=
|
||||
go.n16f.net/thumbhash v1.1.0/go.mod h1:mo9pP7WtfdV9ojIamGFR/Vc0PaPA2l0CUtmYQf/SweU=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
@@ -266,6 +251,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
@@ -277,12 +263,12 @@ golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -299,11 +285,3 @@ gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
|
||||
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
|
||||
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
|
||||
modernc.org/libc v1.72.1 h1:db1xwJ6u1kE3KHTFTTbe2GCrczHPKzlURP0aDC4NGD0=
|
||||
modernc.org/libc v1.72.1/go.mod h1:HRMiC/PhPGLIPM7GzAFCbI+oSgE3dhZ8FWftmRrHVlY=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
|
||||
@@ -20,13 +20,12 @@ type autoscanQueuer = autoscan.Queuer
|
||||
|
||||
// RequestIntegrationLookup adapts the Requests repository to the autoscan
|
||||
// connection resolver's RequestIntegrationLookup: it resolves a soft-linked
|
||||
// Requests integration to its base URL and api key (the repo decrypts the key
|
||||
// on read, so this returns plaintext).
|
||||
// Requests integration to its base URL and (write-only) api-key ref.
|
||||
type RequestIntegrationLookup struct {
|
||||
Repo *mediarequests.Repository
|
||||
}
|
||||
|
||||
func (l RequestIntegrationLookup) Get(ctx context.Context, integrationID string) (baseURL, apiKey string, err error) {
|
||||
func (l RequestIntegrationLookup) Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error) {
|
||||
integration, err := l.Repo.GetIntegration(ctx, integrationID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
@@ -65,8 +64,8 @@ type PluginScanSourceAdapter struct {
|
||||
Svc *plugins.Service
|
||||
}
|
||||
|
||||
func (a PluginScanSourceAdapter) ScanSourceClient(ctx context.Context, pluginID, capabilityID string) (autoscan.PollChangesClient, error) {
|
||||
return a.Svc.ScanSourceClientByPluginID(ctx, pluginID, capabilityID)
|
||||
func (a PluginScanSourceAdapter) ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (autoscan.PollChangesClient, error) {
|
||||
return a.Svc.ScanSourceClient(ctx, installationID, capabilityID)
|
||||
}
|
||||
|
||||
// scanSourceCapabilityType is the plugin capability type autoscan discovery
|
||||
@@ -76,6 +75,15 @@ const scanSourceCapabilityType = "scan_source.v1"
|
||||
// PluginScanSourceLister adapts the plugin installation store to
|
||||
// autoscan.ScanSourceLister: it enumerates every installed scan_source.v1
|
||||
// capability across ALL installed plugins, regardless of enabled state.
|
||||
//
|
||||
// Using List (not ListEnabled) is deliberate for orphan detection: a temporarily
|
||||
// DISABLED-but-installed plugin must NOT be treated as orphaned. If we dropped
|
||||
// disabled plugins here, PollOnce would prune their sources and skip them with no
|
||||
// last_error — they'd vanish silently. Keeping them present means PollOnce still
|
||||
// attempts them; the plugin client fails to load (not running) and the source
|
||||
// gets a visible RecordError instead. Only a fully UNINSTALLED plugin (gone from
|
||||
// List entirely) is a true orphan. The Add-source picker shares this all-installed
|
||||
// set, which is fine — operators can bind sources against an installed capability.
|
||||
type PluginScanSourceLister struct {
|
||||
Store *plugins.InstallationStore
|
||||
}
|
||||
@@ -96,9 +104,10 @@ func (l PluginScanSourceLister) ListScanSources(ctx context.Context) ([]autoscan
|
||||
continue
|
||||
}
|
||||
out = append(out, autoscan.DiscoveredSource{
|
||||
PluginID: inst.PluginID,
|
||||
CapabilityID: c.ID,
|
||||
DisplayName: scanSourceDisplayName(inst.PluginID, c),
|
||||
InstallationID: c.InstallationID,
|
||||
CapabilityID: c.ID,
|
||||
PluginID: inst.PluginID,
|
||||
DisplayName: scanSourceDisplayName(inst.PluginID, c),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -126,22 +135,27 @@ func scanSourceDisplayName(pluginID string, c *plugins.Capability) string {
|
||||
}
|
||||
}
|
||||
|
||||
// AutoscanSecretResolver resolves an encrypted api-key reference to plaintext.
|
||||
// It is satisfied by the server settings repo (Get(ctx, key) (string, error)).
|
||||
type AutoscanSecretResolver interface {
|
||||
Get(ctx context.Context, ref string) (string, error)
|
||||
}
|
||||
|
||||
// BuildAutoscanService wires the v2 autoscan engine from its concrete
|
||||
// dependencies. Both the HTTP router (manual trigger) and the background poll
|
||||
// task share this constructor so the adapter wiring lives in exactly one place.
|
||||
// Credentials are now decrypted inline by the autoscan/requests repos, so there
|
||||
// is no separate secret resolver to thread.
|
||||
func BuildAutoscanService(
|
||||
repo *autoscan.Repository,
|
||||
pluginService *plugins.Service,
|
||||
installationStore *plugins.InstallationStore,
|
||||
requestsRepo *mediarequests.Repository,
|
||||
secrets AutoscanSecretResolver,
|
||||
folderRepo *catalog.FolderRepository,
|
||||
queue autoscanQueuer,
|
||||
redisClient *redis.Client,
|
||||
) *autoscan.Service {
|
||||
provider := autoscan.NewPluginProvider(PluginScanSourceAdapter{pluginService})
|
||||
connRes := autoscan.NewConnectionResolver(RequestIntegrationLookup{requestsRepo})
|
||||
connRes := autoscan.NewConnectionResolver(RequestIntegrationLookup{requestsRepo}, secrets)
|
||||
svc := autoscan.NewService(
|
||||
repo,
|
||||
provider,
|
||||
|
||||
@@ -113,34 +113,20 @@ func NewAdminHandler(
|
||||
|
||||
// createUserRequest represents the JSON body for POST /admin/users.
|
||||
type createUserRequest struct {
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions createStringSliceField `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
}
|
||||
|
||||
type createStringSliceField struct {
|
||||
Set bool
|
||||
Value []string
|
||||
}
|
||||
|
||||
func (f *createStringSliceField) UnmarshalJSON(data []byte) error {
|
||||
f.Set = true
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
f.Value = []string{}
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(data, &f.Value)
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions []string `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
}
|
||||
|
||||
type updateLibraryIDsField struct {
|
||||
@@ -405,11 +391,7 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "max_profiles must be at least 1")
|
||||
return
|
||||
}
|
||||
permissions := auth.DefaultUserPermissions()
|
||||
if req.Permissions.Set {
|
||||
permissions = req.Permissions.Value
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(permissions)
|
||||
permissions, err := auth.NormalizePermissions(req.Permissions)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
@@ -1137,10 +1119,29 @@ func (h *AdminHandler) HandleUpdateItemMetadata(w http.ResponseWriter, r *http.R
|
||||
|
||||
// --- Server Settings endpoints ---
|
||||
|
||||
// sensitiveSettingKeys is the audited allowlist of secret-bearing settings
|
||||
// keys, shared with the at-rest encryption decorator so redaction and
|
||||
// encryption can never drift apart. See catalog.SensitiveSettingKeys.
|
||||
var sensitiveSettingKeys = catalog.SensitiveSettingKeys
|
||||
var sensitiveSettingKeys = map[string]bool{
|
||||
"auth.jwt_secret": true,
|
||||
"s3.public_access_key": true,
|
||||
"s3.public_secret_key": true,
|
||||
"s3.public_token_secret": true,
|
||||
"s3.private_access_key": true,
|
||||
"s3.private_secret_key": true,
|
||||
"s3.user_db_access_key": true,
|
||||
"s3.user_db_secret_key": true,
|
||||
"redis.url": true,
|
||||
"recommendations.openai_api_key": true,
|
||||
"recommendations.embedding_auth_token": true,
|
||||
"tmdb.api_key": true,
|
||||
"introdb.api_key": true,
|
||||
"mdblist.api_key": true,
|
||||
"requests.radarr.api_key": true,
|
||||
"requests.sonarr.api_key": true,
|
||||
"subtitle_ai.api_key": true,
|
||||
"watchsync.trakt.client_id": true,
|
||||
"watchsync.trakt.client_secret": true,
|
||||
"watchsync.simkl.client_id": true,
|
||||
"watchsync.simkl.client_secret": true,
|
||||
}
|
||||
|
||||
// HandleGetSettings handles GET /admin/settings.
|
||||
func (h *AdminHandler) HandleGetSettings(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -1,230 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
// AdminMarkerProvidersHandler serves the per-provider marker config + key
|
||||
// validation API under the RequireAdmin group.
|
||||
type AdminMarkerProvidersHandler struct {
|
||||
Registry *markers.Registry
|
||||
Config *markers.ProviderConfigStore
|
||||
EventBus cache.EventBus
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewAdminMarkerProvidersHandler constructs the handler.
|
||||
func NewAdminMarkerProvidersHandler(registry *markers.Registry, config *markers.ProviderConfigStore, eventBus cache.EventBus, logger *slog.Logger) *AdminMarkerProvidersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &AdminMarkerProvidersHandler{Registry: registry, Config: config, EventBus: eventBus, logger: logger}
|
||||
}
|
||||
|
||||
type providerConfigResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
DisplayName string `json:"display_name,omitempty"`
|
||||
SourceType string `json:"source_type,omitempty"`
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
PluginInstallationID int `json:"plugin_installation_id,omitempty"`
|
||||
CapabilityID string `json:"capability_id,omitempty"`
|
||||
IsSubmitter bool `json:"is_submitter"`
|
||||
FetchEnabled bool `json:"fetch_enabled"`
|
||||
FetchPriority int `json:"fetch_priority"`
|
||||
ContributeEnabled bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
|
||||
type markerUserStatsResponse struct {
|
||||
Total int `json:"total"`
|
||||
Accepted int `json:"accepted"`
|
||||
Pending int `json:"pending"`
|
||||
Rejected int `json:"rejected"`
|
||||
AcceptanceRate float64 `json:"acceptance_rate"`
|
||||
CurrentStreak int `json:"current_streak"`
|
||||
BestStreak int `json:"best_streak"`
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) submitterIDs() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if _, ok := p.(markers.Submitter); ok {
|
||||
out[p.ID()] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) providerDescriptions() map[string]markers.ProviderDescriptor {
|
||||
out := map[string]markers.ProviderDescriptor{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
desc := markers.ProviderDescriptor{ID: p.ID()}
|
||||
if described, ok := p.(markers.DescribedProvider); ok {
|
||||
desc = described.ProviderDescription()
|
||||
}
|
||||
if desc.ID == "" {
|
||||
desc.ID = p.ID()
|
||||
}
|
||||
out[p.ID()] = desc
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HandleListProviders lists registered providers with their config + capability.
|
||||
func (h *AdminMarkerProvidersHandler) HandleListProviders(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
submitters := h.submitterIDs()
|
||||
descriptions := h.providerDescriptions()
|
||||
out := []providerConfigResponse{}
|
||||
if h.Registry != nil {
|
||||
for _, provider := range h.Registry.Providers() {
|
||||
c, ok := h.Config.Get(provider.ID())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
} else {
|
||||
for _, c := range h.Config.List() {
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FetchPriority != out[j].FetchPriority {
|
||||
return out[i].FetchPriority < out[j].FetchPriority
|
||||
}
|
||||
return out[i].Provider < out[j].Provider
|
||||
})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"providers": out})
|
||||
}
|
||||
|
||||
// HandleUpdateProvider updates a provider's config row.
|
||||
func (h *AdminMarkerProvidersHandler) HandleUpdateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider := chi.URLParam(r, "provider")
|
||||
existing, ok := h.Config.Get(provider)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Unknown marker provider")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
FetchEnabled *bool `json:"fetch_enabled"`
|
||||
FetchPriority *int `json:"fetch_priority"`
|
||||
ContributeEnabled *bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal *bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence *float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if body.FetchEnabled != nil {
|
||||
existing.FetchEnabled = *body.FetchEnabled
|
||||
}
|
||||
if body.FetchPriority != nil {
|
||||
existing.FetchPriority = *body.FetchPriority
|
||||
}
|
||||
if body.ContributeEnabled != nil {
|
||||
existing.ContributeEnabled = *body.ContributeEnabled
|
||||
}
|
||||
if body.ContributeAutoLocal != nil {
|
||||
existing.ContributeAutoLocal = *body.ContributeAutoLocal
|
||||
}
|
||||
if body.ContributeMinConfidence != nil {
|
||||
v := *body.ContributeMinConfidence
|
||||
if v < 0 || v > 1 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "contribute_min_confidence must be between 0 and 1")
|
||||
return
|
||||
}
|
||||
existing.ContributeMinConfidence = v
|
||||
}
|
||||
if err := h.Config.Update(r.Context(), existing); err != nil {
|
||||
h.logger.Error("admin markers: update provider config failed", "provider", provider, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update provider")
|
||||
return
|
||||
}
|
||||
if h.EventBus != nil {
|
||||
_ = h.EventBus.Publish(r.Context(), cache.ChannelAdmin, cache.Event{
|
||||
Type: cache.EventMarkerProviderConfigChanged,
|
||||
Payload: provider,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, toProviderConfigResponse(existing, h.submitterIDs()[provider], h.providerDescriptions()[provider]))
|
||||
}
|
||||
|
||||
// HandleValidateProvider validates the provider's configured key and returns stats.
|
||||
func (h *AdminMarkerProvidersHandler) HandleValidateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Registry == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider := chi.URLParam(r, "provider")
|
||||
var submitter markers.Submitter
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if p.ID() == provider {
|
||||
if s, ok := p.(markers.Submitter); ok {
|
||||
submitter = s
|
||||
}
|
||||
}
|
||||
}
|
||||
if submitter == nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Provider does not support contribution")
|
||||
return
|
||||
}
|
||||
stats, err := submitter.FetchUserStats(r.Context())
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": true, "stats": toMarkerUserStatsResponse(stats)})
|
||||
}
|
||||
|
||||
func toProviderConfigResponse(c markers.ProviderConfig, isSubmitter bool, desc markers.ProviderDescriptor) providerConfigResponse {
|
||||
return providerConfigResponse{
|
||||
Provider: c.Provider,
|
||||
DisplayName: desc.DisplayName,
|
||||
SourceType: desc.SourceType,
|
||||
PluginID: desc.PluginID,
|
||||
PluginInstallationID: desc.PluginInstallationID,
|
||||
CapabilityID: desc.CapabilityID,
|
||||
IsSubmitter: isSubmitter,
|
||||
FetchEnabled: c.FetchEnabled,
|
||||
FetchPriority: c.FetchPriority,
|
||||
ContributeEnabled: c.ContributeEnabled,
|
||||
ContributeAutoLocal: c.ContributeAutoLocal,
|
||||
ContributeMinConfidence: c.ContributeMinConfidence,
|
||||
}
|
||||
}
|
||||
|
||||
func toMarkerUserStatsResponse(s markers.UserStats) markerUserStatsResponse {
|
||||
return markerUserStatsResponse{
|
||||
Total: s.Total,
|
||||
Accepted: s.Accepted,
|
||||
Pending: s.Pending,
|
||||
Rejected: s.Rejected,
|
||||
AcceptanceRate: s.AcceptanceRate,
|
||||
CurrentStreak: s.CurrentStreak,
|
||||
BestStreak: s.BestStreak,
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
type fakeMarkerStatsSubmitter struct{}
|
||||
|
||||
func (fakeMarkerStatsSubmitter) ID() string { return "introdb" }
|
||||
func (fakeMarkerStatsSubmitter) FetchMarkers(context.Context, markers.Request) (markers.Result, error) {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) SubmitMarker(context.Context, markers.SubmissionRequest) (markers.SubmissionResult, error) {
|
||||
return markers.SubmissionResult{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) FetchUserStats(context.Context) (markers.UserStats, error) {
|
||||
return markers.UserStats{Total: 10, Accepted: 7, Pending: 2, Rejected: 1, AcceptanceRate: 0.7, CurrentStreak: 3, BestStreak: 5}, nil
|
||||
}
|
||||
|
||||
func TestValidateMarkerProviderUsesSnakeCaseStats(t *testing.T) {
|
||||
reg := markers.NewRegistry(nil)
|
||||
if err := reg.Register(fakeMarkerStatsSubmitter{}); err != nil {
|
||||
t.Fatalf("register provider: %v", err)
|
||||
}
|
||||
h := NewAdminMarkerProvidersHandler(reg, nil, nil, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/markers/providers/introdb/validate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("provider", "introdb")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleValidateProvider(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"acceptance_rate":0.7`) || strings.Contains(body, "AcceptanceRate") {
|
||||
t.Fatalf("unexpected stats response shape: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerProviderResponseIncludesPluginMetadata(t *testing.T) {
|
||||
resp := toProviderConfigResponse(
|
||||
markers.ProviderConfig{Provider: "plugin:4:markers", FetchEnabled: true, FetchPriority: 25},
|
||||
true,
|
||||
markers.ProviderDescriptor{
|
||||
DisplayName: "Plugin Markers",
|
||||
SourceType: markers.ProviderSourcePlugin,
|
||||
PluginID: "silo.plugin.markers",
|
||||
PluginInstallationID: 4,
|
||||
CapabilityID: "markers",
|
||||
},
|
||||
)
|
||||
if resp.DisplayName != "Plugin Markers" || resp.SourceType != markers.ProviderSourcePlugin {
|
||||
t.Fatalf("plugin metadata fields missing: %+v", resp)
|
||||
}
|
||||
if resp.PluginID != "silo.plugin.markers" ||
|
||||
resp.PluginInstallationID != 4 ||
|
||||
resp.CapabilityID != "markers" ||
|
||||
!resp.IsSubmitter {
|
||||
t.Fatalf("plugin identity fields missing: %+v", resp)
|
||||
}
|
||||
}
|
||||
@@ -22,9 +22,7 @@ type AdminStats struct {
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalUsers int `json:"total_users"`
|
||||
TotalMovies int `json:"total_movies"`
|
||||
TotalMovieFiles int `json:"total_movie_files"`
|
||||
TotalShows int `json:"total_shows"`
|
||||
TotalShowFiles int `json:"total_show_files"`
|
||||
ActiveStreams int `json:"active_streams"`
|
||||
TotalStorageBytes int64 `json:"total_storage_bytes"`
|
||||
WatchProviderActivity WatchProviderActivity `json:"watch_provider_activity"`
|
||||
@@ -129,15 +127,13 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
}
|
||||
|
||||
var (
|
||||
totalUsers int64
|
||||
totalItems int64
|
||||
totalFiles int64
|
||||
totalMovies int64
|
||||
totalMovieFiles int64
|
||||
totalShows int64
|
||||
totalShowFiles int64
|
||||
activeStreams int64
|
||||
totalStorage int64
|
||||
totalUsers int64
|
||||
totalItems int64
|
||||
totalFiles int64
|
||||
totalMovies int64
|
||||
totalShows int64
|
||||
activeStreams int64
|
||||
totalStorage int64
|
||||
)
|
||||
|
||||
row := pool.QueryRow(ctx, `
|
||||
@@ -155,34 +151,8 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
file_stats AS (
|
||||
SELECT
|
||||
COUNT(*)::bigint AS total_files,
|
||||
COUNT(*) FILTER (WHERE file_kind = 'movie')::bigint AS total_movie_files,
|
||||
COUNT(*) FILTER (WHERE file_kind = 'series')::bigint AS total_show_files,
|
||||
COALESCE(SUM(file_size), 0)::bigint AS total_storage_bytes
|
||||
FROM (
|
||||
SELECT
|
||||
media_files.file_size,
|
||||
CASE
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_items.type, ''), ''))) = 'movie'
|
||||
THEN 'movie'
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_items.type, ''), ''))) = 'series'
|
||||
THEN 'series'
|
||||
WHEN episodes.content_id IS NOT NULL
|
||||
THEN 'series'
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_files.base_type, ''), ''))) IN ('movie', 'movies')
|
||||
THEN 'movie'
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_files.base_type, ''), ''))) IN ('series', 'tv', 'show', 'shows', 'tvshows')
|
||||
THEN 'series'
|
||||
WHEN lower(trim(media_folders.type)) IN ('movie', 'movies')
|
||||
THEN 'movie'
|
||||
WHEN lower(trim(media_folders.type)) IN ('series', 'tv', 'show', 'shows', 'tvshows')
|
||||
THEN 'series'
|
||||
ELSE ''
|
||||
END AS file_kind
|
||||
FROM media_files
|
||||
JOIN media_folders ON media_folders.id = media_files.media_folder_id
|
||||
LEFT JOIN media_items ON media_items.content_id = media_files.content_id
|
||||
LEFT JOIN episodes ON episodes.content_id = media_files.episode_id
|
||||
) classified_files
|
||||
FROM media_files
|
||||
),
|
||||
session_stats AS (
|
||||
SELECT COUNT(*)::bigint AS active_streams
|
||||
@@ -193,9 +163,7 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
item_stats.total_items,
|
||||
file_stats.total_files,
|
||||
item_stats.total_movies,
|
||||
file_stats.total_movie_files,
|
||||
item_stats.total_shows,
|
||||
file_stats.total_show_files,
|
||||
session_stats.active_streams,
|
||||
file_stats.total_storage_bytes
|
||||
FROM user_stats
|
||||
@@ -208,9 +176,7 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
&totalItems,
|
||||
&totalFiles,
|
||||
&totalMovies,
|
||||
&totalMovieFiles,
|
||||
&totalShows,
|
||||
&totalShowFiles,
|
||||
&activeStreams,
|
||||
&totalStorage,
|
||||
); err != nil {
|
||||
@@ -228,9 +194,7 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
TotalItems: int(totalItems),
|
||||
TotalFiles: int(totalFiles),
|
||||
TotalMovies: int(totalMovies),
|
||||
TotalMovieFiles: int(totalMovieFiles),
|
||||
TotalShows: int(totalShows),
|
||||
TotalShowFiles: int(totalShowFiles),
|
||||
ActiveStreams: int(activeStreams),
|
||||
TotalStorageBytes: totalStorage,
|
||||
WatchProviderActivity: activity,
|
||||
|
||||
@@ -1,757 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/oklog/ulid/v2"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
const audioPlaybackSessionTTL = 12 * time.Hour
|
||||
|
||||
type AudioDetailService interface {
|
||||
GetItemDetail(ctx context.Context, contentID string, filter catalog.AccessFilter) (*catalog.ItemDetail, error)
|
||||
}
|
||||
|
||||
type AudioFileResolver interface {
|
||||
GetByContentID(ctx context.Context, contentID string) ([]*models.MediaFile, error)
|
||||
GetByID(ctx context.Context, id int) (*models.MediaFile, error)
|
||||
}
|
||||
|
||||
type AudioHandler struct {
|
||||
detailSvc AudioDetailService
|
||||
files AudioFileResolver
|
||||
storeProvider userstore.UserStoreProvider
|
||||
bookmarkStore *PGAudioBookmarkStore
|
||||
|
||||
mu sync.Mutex
|
||||
sessions map[string]audioPlaybackSession
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewAudioHandler(
|
||||
detailSvc AudioDetailService,
|
||||
files AudioFileResolver,
|
||||
storeProvider userstore.UserStoreProvider,
|
||||
bookmarkStore *PGAudioBookmarkStore,
|
||||
) *AudioHandler {
|
||||
return &AudioHandler{
|
||||
detailSvc: detailSvc,
|
||||
files: files,
|
||||
storeProvider: storeProvider,
|
||||
bookmarkStore: bookmarkStore,
|
||||
sessions: make(map[string]audioPlaybackSession),
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
}
|
||||
|
||||
type audioPlaybackSession struct {
|
||||
ID string
|
||||
UserID int
|
||||
ProfileID string
|
||||
ContentID string
|
||||
TotalSeconds float64
|
||||
Files []audioSessionFile
|
||||
CreatedAt time.Time
|
||||
LastSeenAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
type audioSessionFile struct {
|
||||
FileID int
|
||||
FilePath string
|
||||
Start float64
|
||||
Duration float64
|
||||
Mode string
|
||||
TrackIdx int
|
||||
}
|
||||
|
||||
type audioStartPlaybackRequest struct {
|
||||
ContentID string `json:"content_id"`
|
||||
StartPosition *float64 `json:"start_position,omitempty"`
|
||||
Restart bool `json:"restart,omitempty"`
|
||||
}
|
||||
|
||||
type audioProgressRequest struct {
|
||||
Position float64 `json:"position"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
IsPaused bool `json:"is_paused"`
|
||||
}
|
||||
|
||||
type audioPlaybackResponse struct {
|
||||
SessionID string `json:"session_id"`
|
||||
ContentID string `json:"content_id"`
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Subtitle string `json:"subtitle,omitempty"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
BackdropURL string `json:"backdrop_url,omitempty"`
|
||||
TotalDurationSeconds float64 `json:"total_duration_seconds"`
|
||||
ResumePositionSeconds float64 `json:"resume_position_seconds"`
|
||||
Tracks []audioTrackResponse `json:"tracks"`
|
||||
Chapters []audioChapter `json:"chapters"`
|
||||
Bookmarks []audioBookmark `json:"bookmarks"`
|
||||
}
|
||||
|
||||
type audioTrackResponse struct {
|
||||
Index int `json:"index"`
|
||||
FileID int `json:"file_id"`
|
||||
FileName string `json:"file_name,omitempty"`
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
StartOffsetSeconds float64 `json:"start_offset_seconds"`
|
||||
StreamURL string `json:"stream_url"`
|
||||
StreamType string `json:"stream_type"`
|
||||
PlayMethod string `json:"play_method"`
|
||||
Codec string `json:"codec,omitempty"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
Chapters []audioChapter `json:"chapters"`
|
||||
}
|
||||
|
||||
type audioChapter struct {
|
||||
Index int `json:"index"`
|
||||
Title string `json:"title,omitempty"`
|
||||
StartSeconds float64 `json:"start_seconds"`
|
||||
EndSeconds float64 `json:"end_seconds,omitempty"`
|
||||
TrackIndex int `json:"track_index,omitempty"`
|
||||
}
|
||||
|
||||
type audioBookmark struct {
|
||||
ID string `json:"id"`
|
||||
ContentID string `json:"content_id"`
|
||||
TimeSeconds float64 `json:"time_seconds"`
|
||||
Title string `json:"title"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
UpdatedAt string `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type audioBookmarkRequest struct {
|
||||
TimeSeconds *float64 `json:"time_seconds"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.detailSvc == nil || h.files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Audio playback is not configured")
|
||||
return
|
||||
}
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
if userID == 0 || profileID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Profile is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req audioStartPlaybackRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
req.ContentID = strings.TrimSpace(req.ContentID)
|
||||
if req.ContentID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "content_id is required")
|
||||
return
|
||||
}
|
||||
|
||||
filter := requestAccessFilter(r)
|
||||
detail, err := h.detailSvc.GetItemDetail(r.Context(), req.ContentID, filter)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item detail")
|
||||
return
|
||||
}
|
||||
if detail == nil || detail.Type != "audiobook" {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Audiobook not found")
|
||||
return
|
||||
}
|
||||
|
||||
files, err := h.files.GetByContentID(r.Context(), req.ContentID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load audio files")
|
||||
return
|
||||
}
|
||||
files = catalog.FilterMediaFilesByAccess(files, filter)
|
||||
sortAudioFiles(files)
|
||||
if len(files) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "No playable audio files found")
|
||||
return
|
||||
}
|
||||
|
||||
sessionID := ulid.Make().String()
|
||||
tracks, sessionFiles, chapters, totalDuration := buildAudioTrackResponses(sessionID, files)
|
||||
if detail.Audiobook != nil && detail.Audiobook.TotalDurationSeconds > 0 {
|
||||
totalDuration = float64(detail.Audiobook.TotalDurationSeconds)
|
||||
}
|
||||
|
||||
resume := resolveAudioResumePosition(r.Context(), h.storeProvider, userID, profileID, req.ContentID)
|
||||
if req.Restart {
|
||||
resume = 0
|
||||
} else if req.StartPosition != nil && isFiniteNonNegative(*req.StartPosition) {
|
||||
resume = clampFloat(*req.StartPosition, 0, totalDuration)
|
||||
}
|
||||
|
||||
bookmarks, _ := h.listBookmarks(r.Context(), userID, profileID, req.ContentID)
|
||||
session := audioPlaybackSession{
|
||||
ID: sessionID,
|
||||
UserID: userID,
|
||||
ProfileID: profileID,
|
||||
ContentID: req.ContentID,
|
||||
TotalSeconds: totalDuration,
|
||||
Files: sessionFiles,
|
||||
CreatedAt: h.now(),
|
||||
LastSeenAt: h.now(),
|
||||
ExpiresAt: h.now().Add(audioPlaybackSessionTTL),
|
||||
}
|
||||
h.putSession(session)
|
||||
|
||||
writeJSON(w, http.StatusOK, audioPlaybackResponse{
|
||||
SessionID: sessionID,
|
||||
ContentID: detail.ContentID,
|
||||
Type: detail.Type,
|
||||
Title: detail.Title,
|
||||
Subtitle: audioSubtitle(detail),
|
||||
PosterURL: detail.PosterURL,
|
||||
BackdropURL: detail.BackdropURL,
|
||||
TotalDurationSeconds: totalDuration,
|
||||
ResumePositionSeconds: resume,
|
||||
Tracks: tracks,
|
||||
Chapters: chapters,
|
||||
Bookmarks: bookmarks,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleSyncPlayback(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := h.sessionForRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req audioProgressRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
duration := req.Duration
|
||||
if duration <= 0 {
|
||||
duration = session.TotalSeconds
|
||||
}
|
||||
position := clampFloat(req.Position, 0, duration)
|
||||
if err := h.persistProgress(r.Context(), session.UserID, session.ProfileID, session.ContentID, position, duration); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to sync audio progress")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"session_id": session.ID,
|
||||
"position": position,
|
||||
"duration": duration,
|
||||
"is_paused": req.IsPaused,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleClosePlayback(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := h.sessionForRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req audioProgressRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if isFiniteNonNegative(req.Position) {
|
||||
duration := req.Duration
|
||||
if duration <= 0 {
|
||||
duration = session.TotalSeconds
|
||||
}
|
||||
position := clampFloat(req.Position, 0, duration)
|
||||
_ = h.persistProgress(r.Context(), session.UserID, session.ProfileID, session.ContentID, position, duration)
|
||||
}
|
||||
h.deleteSession(session.ID)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleStreamTrack(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := h.sessionForRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rawIndex := chi.URLParam(r, "track_index")
|
||||
index, err := strconv.Atoi(rawIndex)
|
||||
if err != nil || index < 0 || index >= len(session.Files) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid track index")
|
||||
return
|
||||
}
|
||||
file := session.Files[index]
|
||||
mode := r.URL.Query().Get("mode")
|
||||
if mode == "" {
|
||||
mode = file.Mode
|
||||
}
|
||||
if mode == "aac" {
|
||||
seek := 0.0
|
||||
if raw := r.URL.Query().Get("seek"); raw != "" {
|
||||
if parsed, err := strconv.ParseFloat(raw, 64); err == nil && parsed >= 0 {
|
||||
seek = parsed
|
||||
}
|
||||
}
|
||||
if err := playback.ServeRemux(w, r, file.FilePath, "mp4", seek, true, 0); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
_ = playback.ServeDirectPlay(w, r, file.FilePath)
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleListBookmarks(w http.ResponseWriter, r *http.Request) {
|
||||
contentID, ok := h.contentIDParam(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
bookmarks, err := h.listBookmarks(r.Context(), userID, profileID, contentID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list bookmarks")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"bookmarks": bookmarks})
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleCreateBookmark(w http.ResponseWriter, r *http.Request) {
|
||||
contentID, ok := h.contentIDParam(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.bookmarkStore == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Bookmarks are not configured")
|
||||
return
|
||||
}
|
||||
var req audioBookmarkRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.TimeSeconds == nil || !isFiniteNonNegative(*req.TimeSeconds) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "time_seconds is required")
|
||||
return
|
||||
}
|
||||
filter := requestAccessFilter(r)
|
||||
detail, err := h.detailSvc.GetItemDetail(r.Context(), contentID, filter)
|
||||
if err != nil || detail == nil || detail.Type != "audiobook" {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Audiobook not found")
|
||||
return
|
||||
}
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
bookmark, err := h.bookmarkStore.Upsert(r.Context(), userID, profileID, contentID, *req.TimeSeconds, req.Title)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save bookmark")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, bookmark)
|
||||
}
|
||||
|
||||
func (h *AudioHandler) HandleDeleteBookmark(w http.ResponseWriter, r *http.Request) {
|
||||
contentID, ok := h.contentIDParam(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.bookmarkStore == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Bookmarks are not configured")
|
||||
return
|
||||
}
|
||||
rawTime := chi.URLParam(r, "time_seconds")
|
||||
timeSeconds, err := strconv.ParseFloat(rawTime, 64)
|
||||
if err != nil || !isFiniteNonNegative(timeSeconds) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid bookmark time")
|
||||
return
|
||||
}
|
||||
if err := h.bookmarkStore.Delete(r.Context(), apimw.GetUserID(r.Context()), apimw.GetProfileID(r.Context()), contentID, timeSeconds); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete bookmark")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *AudioHandler) contentIDParam(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
contentID := strings.TrimSpace(chi.URLParam(r, "content_id"))
|
||||
if contentID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "content_id is required")
|
||||
return "", false
|
||||
}
|
||||
return contentID, true
|
||||
}
|
||||
|
||||
func (h *AudioHandler) sessionForRequest(w http.ResponseWriter, r *http.Request) (audioPlaybackSession, bool) {
|
||||
sessionID := chi.URLParam(r, "session_id")
|
||||
if sessionID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Session ID is required")
|
||||
return audioPlaybackSession{}, false
|
||||
}
|
||||
session, ok := h.getSession(sessionID)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Audio playback session not found")
|
||||
return audioPlaybackSession{}, false
|
||||
}
|
||||
if session.UserID != apimw.GetUserID(r.Context()) {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user")
|
||||
return audioPlaybackSession{}, false
|
||||
}
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
if profileID != "" && session.ProfileID != profileID {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another profile")
|
||||
return audioPlaybackSession{}, false
|
||||
}
|
||||
return session, true
|
||||
}
|
||||
|
||||
func (h *AudioHandler) putSession(session audioPlaybackSession) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.pruneExpiredLocked(h.now())
|
||||
h.sessions[session.ID] = session
|
||||
}
|
||||
|
||||
func (h *AudioHandler) getSession(sessionID string) (audioPlaybackSession, bool) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.now()
|
||||
h.pruneExpiredLocked(now)
|
||||
session, ok := h.sessions[sessionID]
|
||||
if !ok {
|
||||
return audioPlaybackSession{}, false
|
||||
}
|
||||
if now.After(session.ExpiresAt) {
|
||||
delete(h.sessions, sessionID)
|
||||
return audioPlaybackSession{}, false
|
||||
}
|
||||
session.LastSeenAt = now
|
||||
h.sessions[sessionID] = session
|
||||
return session, true
|
||||
}
|
||||
|
||||
func (h *AudioHandler) deleteSession(sessionID string) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
delete(h.sessions, sessionID)
|
||||
}
|
||||
|
||||
func (h *AudioHandler) pruneExpiredLocked(now time.Time) {
|
||||
for id, session := range h.sessions {
|
||||
if now.After(session.ExpiresAt) {
|
||||
delete(h.sessions, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AudioHandler) persistProgress(ctx context.Context, userID int, profileID, contentID string, position, duration float64) error {
|
||||
if h.storeProvider == nil {
|
||||
return nil
|
||||
}
|
||||
store, err := h.storeProvider.ForUser(ctx, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.SetProgress(ctx, profileID, contentID, position, duration, userstore.ProgressThresholds{})
|
||||
}
|
||||
|
||||
func (h *AudioHandler) listBookmarks(ctx context.Context, userID int, profileID, contentID string) ([]audioBookmark, error) {
|
||||
if h.bookmarkStore == nil {
|
||||
return []audioBookmark{}, nil
|
||||
}
|
||||
return h.bookmarkStore.List(ctx, userID, profileID, contentID)
|
||||
}
|
||||
|
||||
func buildAudioTrackResponses(sessionID string, files []*models.MediaFile) ([]audioTrackResponse, []audioSessionFile, []audioChapter, float64) {
|
||||
tracks := make([]audioTrackResponse, 0, len(files))
|
||||
sessionFiles := make([]audioSessionFile, 0, len(files))
|
||||
allChapters := []audioChapter{}
|
||||
offset := 0.0
|
||||
for index, file := range files {
|
||||
duration := float64(file.Duration)
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
mode := audioDeliveryMode(file)
|
||||
streamURL := fmt.Sprintf("/api/v1/audio/playback/%s/tracks/%d", sessionID, index)
|
||||
if mode == "aac" {
|
||||
streamURL += "?mode=aac"
|
||||
}
|
||||
chapters := audioChaptersForTrack(index, offset, file.Chapters)
|
||||
allChapters = append(allChapters, chapters...)
|
||||
tracks = append(tracks, audioTrackResponse{
|
||||
Index: index,
|
||||
FileID: file.ID,
|
||||
FileName: filepath.Base(file.FilePath),
|
||||
DurationSeconds: duration,
|
||||
StartOffsetSeconds: offset,
|
||||
StreamURL: streamURL,
|
||||
StreamType: "progressive",
|
||||
PlayMethod: mode,
|
||||
Codec: file.CodecAudio,
|
||||
Container: file.Container,
|
||||
Bitrate: file.Bitrate,
|
||||
Chapters: chapters,
|
||||
})
|
||||
sessionFiles = append(sessionFiles, audioSessionFile{
|
||||
FileID: file.ID,
|
||||
FilePath: file.FilePath,
|
||||
Start: offset,
|
||||
Duration: duration,
|
||||
Mode: mode,
|
||||
TrackIdx: index,
|
||||
})
|
||||
offset += duration
|
||||
}
|
||||
return tracks, sessionFiles, allChapters, offset
|
||||
}
|
||||
|
||||
func audioChaptersForTrack(trackIndex int, offset float64, chapters []models.MediaChapter) []audioChapter {
|
||||
out := make([]audioChapter, 0, len(chapters))
|
||||
for _, chapter := range chapters {
|
||||
start := offset + maxFloat(0, chapter.StartSeconds)
|
||||
end := 0.0
|
||||
if chapter.EndSeconds > 0 {
|
||||
end = offset + chapter.EndSeconds
|
||||
}
|
||||
out = append(out, audioChapter{
|
||||
Index: len(out),
|
||||
Title: chapter.Title,
|
||||
StartSeconds: start,
|
||||
EndSeconds: end,
|
||||
TrackIndex: trackIndex,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortAudioFiles(files []*models.MediaFile) {
|
||||
sort.SliceStable(files, func(i, j int) bool {
|
||||
left, right := files[i], files[j]
|
||||
if left.PresentationPartIndex != right.PresentationPartIndex {
|
||||
if left.PresentationPartIndex == 0 {
|
||||
return false
|
||||
}
|
||||
if right.PresentationPartIndex == 0 {
|
||||
return true
|
||||
}
|
||||
return left.PresentationPartIndex < right.PresentationPartIndex
|
||||
}
|
||||
if left.FilePath != right.FilePath {
|
||||
return left.FilePath < right.FilePath
|
||||
}
|
||||
return left.ID < right.ID
|
||||
})
|
||||
}
|
||||
|
||||
func audioDeliveryMode(file *models.MediaFile) string {
|
||||
if file == nil {
|
||||
return "direct"
|
||||
}
|
||||
codec := strings.ToLower(strings.TrimSpace(file.CodecAudio))
|
||||
container := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(file.Container), "."))
|
||||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(file.FilePath), "."))
|
||||
if container == "" {
|
||||
container = ext
|
||||
}
|
||||
switch codec {
|
||||
case "aac", "mp3", "alac", "flac":
|
||||
return "direct"
|
||||
case "":
|
||||
switch container {
|
||||
case "mp3", "m4a", "m4b", "mp4", "aac", "flac", "wav", "aiff", "aif":
|
||||
return "direct"
|
||||
}
|
||||
}
|
||||
return "aac"
|
||||
}
|
||||
|
||||
func audioSubtitle(detail *catalog.ItemDetail) string {
|
||||
if detail == nil || detail.Audiobook == nil {
|
||||
return ""
|
||||
}
|
||||
parts := []string{}
|
||||
if len(detail.Audiobook.Authors) > 0 {
|
||||
names := make([]string, 0, len(detail.Audiobook.Authors))
|
||||
for _, author := range detail.Audiobook.Authors {
|
||||
if strings.TrimSpace(author.Name) != "" {
|
||||
names = append(names, author.Name)
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
parts = append(parts, strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
if len(detail.Audiobook.Narrators) > 0 {
|
||||
names := make([]string, 0, len(detail.Audiobook.Narrators))
|
||||
for _, narrator := range detail.Audiobook.Narrators {
|
||||
if strings.TrimSpace(narrator.Name) != "" {
|
||||
names = append(names, narrator.Name)
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
parts = append(parts, "Narrated by "+strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " • ")
|
||||
}
|
||||
|
||||
func resolveAudioResumePosition(ctx context.Context, provider userstore.UserStoreProvider, userID int, profileID, contentID string) float64 {
|
||||
if provider == nil || userID == 0 || profileID == "" {
|
||||
return 0
|
||||
}
|
||||
store, err := provider.ForUser(ctx, userID)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
progress, err := store.GetProgress(ctx, profileID, contentID)
|
||||
if err != nil || progress == nil || progress.Completed {
|
||||
return 0
|
||||
}
|
||||
if progress.PositionSeconds < 30 {
|
||||
return 0
|
||||
}
|
||||
if progress.DurationSeconds > 0 && progress.PositionSeconds >= progress.DurationSeconds-5 {
|
||||
return 0
|
||||
}
|
||||
return progress.PositionSeconds
|
||||
}
|
||||
|
||||
func isFiniteNonNegative(value float64) bool {
|
||||
return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0
|
||||
}
|
||||
|
||||
func clampFloat(value, lower, upper float64) float64 {
|
||||
if !isFiniteNonNegative(value) {
|
||||
return lower
|
||||
}
|
||||
if value < lower {
|
||||
return lower
|
||||
}
|
||||
if upper > lower && value > upper {
|
||||
return upper
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func maxFloat(a, b float64) float64 {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
type PGAudioBookmarkStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewPGAudioBookmarkStore(pool *pgxpool.Pool) *PGAudioBookmarkStore {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return &PGAudioBookmarkStore{Pool: pool}
|
||||
}
|
||||
|
||||
func (s *PGAudioBookmarkStore) List(ctx context.Context, userID int, profileID, contentID string) ([]audioBookmark, error) {
|
||||
if s == nil || s.Pool == nil {
|
||||
return []audioBookmark{}, nil
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, library_item_id, time_seconds, title, created_at, updated_at
|
||||
FROM abs_bookmarks
|
||||
WHERE user_id = $1
|
||||
AND COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid)
|
||||
= COALESCE($2::uuid, '00000000-0000-0000-0000-000000000000'::uuid)
|
||||
AND library_item_id = $3
|
||||
ORDER BY time_seconds ASC`,
|
||||
userID, profileArg(profileID), contentID,
|
||||
)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return []audioBookmark{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
bookmarks := []audioBookmark{}
|
||||
for rows.Next() {
|
||||
var createdAt, updatedAt time.Time
|
||||
var bookmark audioBookmark
|
||||
if err := rows.Scan(&bookmark.ID, &bookmark.ContentID, &bookmark.TimeSeconds, &bookmark.Title, &createdAt, &updatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bookmark.CreatedAt = createdAt.UTC().Format(time.RFC3339)
|
||||
bookmark.UpdatedAt = updatedAt.UTC().Format(time.RFC3339)
|
||||
bookmarks = append(bookmarks, bookmark)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bookmarks, nil
|
||||
}
|
||||
|
||||
func (s *PGAudioBookmarkStore) Upsert(ctx context.Context, userID int, profileID, contentID string, timeSeconds float64, title string) (audioBookmark, error) {
|
||||
if s == nil || s.Pool == nil {
|
||||
return audioBookmark{}, fmt.Errorf("bookmark store unavailable")
|
||||
}
|
||||
id := ulid.Make().String()
|
||||
var createdAt, updatedAt time.Time
|
||||
var bookmark audioBookmark
|
||||
err := s.Pool.QueryRow(ctx, `
|
||||
INSERT INTO abs_bookmarks
|
||||
(id, user_id, profile_id, library_item_id, time_seconds, title)
|
||||
VALUES ($1, $2, $3::uuid, $4, $5, $6)
|
||||
ON CONFLICT (
|
||||
user_id,
|
||||
COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid),
|
||||
library_item_id,
|
||||
time_seconds
|
||||
) DO UPDATE
|
||||
SET title = EXCLUDED.title,
|
||||
updated_at = now()
|
||||
RETURNING id, library_item_id, time_seconds, title, created_at, updated_at`,
|
||||
id, userID, profileArg(profileID), contentID, timeSeconds, strings.TrimSpace(title),
|
||||
).Scan(&bookmark.ID, &bookmark.ContentID, &bookmark.TimeSeconds, &bookmark.Title, &createdAt, &updatedAt)
|
||||
if err != nil {
|
||||
return audioBookmark{}, err
|
||||
}
|
||||
bookmark.CreatedAt = createdAt.UTC().Format(time.RFC3339)
|
||||
bookmark.UpdatedAt = updatedAt.UTC().Format(time.RFC3339)
|
||||
return bookmark, nil
|
||||
}
|
||||
|
||||
func (s *PGAudioBookmarkStore) Delete(ctx context.Context, userID int, profileID, contentID string, timeSeconds float64) error {
|
||||
if s == nil || s.Pool == nil {
|
||||
return fmt.Errorf("bookmark store unavailable")
|
||||
}
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
DELETE FROM abs_bookmarks
|
||||
WHERE user_id = $1
|
||||
AND COALESCE(profile_id, '00000000-0000-0000-0000-000000000000'::uuid)
|
||||
= COALESCE($2::uuid, '00000000-0000-0000-0000-000000000000'::uuid)
|
||||
AND library_item_id = $3
|
||||
AND time_seconds = $4`,
|
||||
userID, profileArg(profileID), contentID, timeSeconds,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func profileArg(profileID string) any {
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return nil
|
||||
}
|
||||
return profileID
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type stubAudioDetailService struct {
|
||||
detail *catalog.ItemDetail
|
||||
}
|
||||
|
||||
func (s stubAudioDetailService) GetItemDetail(context.Context, string, catalog.AccessFilter) (*catalog.ItemDetail, error) {
|
||||
return s.detail, nil
|
||||
}
|
||||
|
||||
type stubAudioFiles struct {
|
||||
files []*models.MediaFile
|
||||
}
|
||||
|
||||
func (s stubAudioFiles) GetByContentID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return append([]*models.MediaFile(nil), s.files...), nil
|
||||
}
|
||||
|
||||
func (s stubAudioFiles) GetByID(_ context.Context, id int) (*models.MediaFile, error) {
|
||||
for _, file := range s.files {
|
||||
if file.ID == id {
|
||||
return file, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestAudioStartPlaybackOrdersTracksAndBuildsGlobalChapters(t *testing.T) {
|
||||
handler := NewAudioHandler(
|
||||
stubAudioDetailService{detail: testAudiobookDetail()},
|
||||
stubAudioFiles{files: []*models.MediaFile{
|
||||
testAudioFile(3, "part-03.mp3", 30, 30),
|
||||
testAudioFile(1, "part-01.mp3", 10, 10),
|
||||
testAudioFile(2, "part-02.mp3", 20, 20),
|
||||
}},
|
||||
testUserStoreProvider{store: newPlaybackTestStore(t)},
|
||||
nil,
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/audio/playback/start", strings.NewReader(`{"content_id":"book-1"}`))
|
||||
req = req.WithContext(newAuthorizedPlaybackContext())
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleStartPlayback(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var resp audioPlaybackResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Tracks) != 3 {
|
||||
t.Fatalf("tracks = %d, want 3", len(resp.Tracks))
|
||||
}
|
||||
if resp.Tracks[0].FileID != 1 || resp.Tracks[1].FileID != 2 || resp.Tracks[2].FileID != 3 {
|
||||
t.Fatalf("track order = [%d %d %d], want [1 2 3]", resp.Tracks[0].FileID, resp.Tracks[1].FileID, resp.Tracks[2].FileID)
|
||||
}
|
||||
if resp.Tracks[1].StartOffsetSeconds != 10 || resp.Tracks[2].StartOffsetSeconds != 30 {
|
||||
t.Fatalf("offsets = %.0f %.0f, want 10 30", resp.Tracks[1].StartOffsetSeconds, resp.Tracks[2].StartOffsetSeconds)
|
||||
}
|
||||
if len(resp.Chapters) != 3 || resp.Chapters[2].StartSeconds != 30 {
|
||||
t.Fatalf("global chapters = %+v, want third chapter at 30s", resp.Chapters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudioSyncPersistsCumulativeProgressAndEnforcesOwner(t *testing.T) {
|
||||
store := newPlaybackTestStore(t)
|
||||
handler := NewAudioHandler(
|
||||
stubAudioDetailService{detail: testAudiobookDetail()},
|
||||
stubAudioFiles{files: []*models.MediaFile{
|
||||
testAudioFile(1, "part-01.mp3", 10, 10),
|
||||
testAudioFile(2, "part-02.mp3", 20, 20),
|
||||
}},
|
||||
testUserStoreProvider{store: store},
|
||||
nil,
|
||||
)
|
||||
|
||||
startReq := httptest.NewRequest(http.MethodPost, "/api/v1/audio/playback/start", strings.NewReader(`{"content_id":"book-1"}`))
|
||||
startReq = startReq.WithContext(newAuthorizedPlaybackContext())
|
||||
startRec := httptest.NewRecorder()
|
||||
handler.HandleStartPlayback(startRec, startReq)
|
||||
if startRec.Code != http.StatusOK {
|
||||
t.Fatalf("start status = %d body=%s", startRec.Code, startRec.Body.String())
|
||||
}
|
||||
var start audioPlaybackResponse
|
||||
if err := json.Unmarshal(startRec.Body.Bytes(), &start); err != nil {
|
||||
t.Fatalf("decode start: %v", err)
|
||||
}
|
||||
|
||||
otherCtx := apimw.SetClaims(context.Background(), &auth.Claims{UserID: 2, Role: "user", TokenType: auth.TokenTypeAccess})
|
||||
otherCtx = apimw.SetProfileID(otherCtx, "profile-1")
|
||||
denied := httptest.NewRequest(http.MethodPatch, "/api/v1/audio/playback/"+start.SessionID+"/sync", strings.NewReader(`{"position":12,"duration":30}`))
|
||||
denied = withAudioRouteParam(denied.WithContext(otherCtx), "session_id", start.SessionID)
|
||||
deniedRec := httptest.NewRecorder()
|
||||
handler.HandleSyncPlayback(deniedRec, denied)
|
||||
if deniedRec.Code != http.StatusForbidden {
|
||||
t.Fatalf("foreign sync status = %d, want 403", deniedRec.Code)
|
||||
}
|
||||
|
||||
syncReq := httptest.NewRequest(http.MethodPatch, "/api/v1/audio/playback/"+start.SessionID+"/sync", strings.NewReader(`{"position":18,"duration":30}`))
|
||||
syncReq = withAudioRouteParam(syncReq.WithContext(newAuthorizedPlaybackContext()), "session_id", start.SessionID)
|
||||
syncRec := httptest.NewRecorder()
|
||||
handler.HandleSyncPlayback(syncRec, syncReq)
|
||||
if syncRec.Code != http.StatusOK {
|
||||
t.Fatalf("sync status = %d body=%s", syncRec.Code, syncRec.Body.String())
|
||||
}
|
||||
|
||||
progress, err := store.GetProgress(context.Background(), "profile-1", "book-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get progress: %v", err)
|
||||
}
|
||||
if progress == nil || progress.PositionSeconds != 18 || progress.DurationSeconds != 30 {
|
||||
t.Fatalf("progress = %+v, want 18/30", progress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudioSessionExpires(t *testing.T) {
|
||||
handler := NewAudioHandler(
|
||||
stubAudioDetailService{detail: testAudiobookDetail()},
|
||||
stubAudioFiles{files: []*models.MediaFile{testAudioFile(1, "part-01.mp3", 10, 10)}},
|
||||
testUserStoreProvider{store: newPlaybackTestStore(t)},
|
||||
nil,
|
||||
)
|
||||
now := time.Date(2026, 6, 8, 12, 0, 0, 0, time.UTC)
|
||||
handler.now = func() time.Time { return now }
|
||||
|
||||
startReq := httptest.NewRequest(http.MethodPost, "/api/v1/audio/playback/start", strings.NewReader(`{"content_id":"book-1"}`))
|
||||
startReq = startReq.WithContext(newAuthorizedPlaybackContext())
|
||||
startRec := httptest.NewRecorder()
|
||||
handler.HandleStartPlayback(startRec, startReq)
|
||||
var start audioPlaybackResponse
|
||||
if err := json.Unmarshal(startRec.Body.Bytes(), &start); err != nil {
|
||||
t.Fatalf("decode start: %v", err)
|
||||
}
|
||||
|
||||
now = now.Add(audioPlaybackSessionTTL + time.Second)
|
||||
streamReq := httptest.NewRequest(http.MethodHead, "/api/v1/audio/playback/"+start.SessionID+"/tracks/0", nil)
|
||||
streamReq = withAudioRouteParam(streamReq.WithContext(newAuthorizedPlaybackContext()), "session_id", start.SessionID)
|
||||
streamReq = withAudioRouteParam(streamReq, "track_index", "0")
|
||||
streamRec := httptest.NewRecorder()
|
||||
handler.HandleStreamTrack(streamRec, streamReq)
|
||||
if streamRec.Code != http.StatusNotFound {
|
||||
t.Fatalf("expired stream status = %d, want 404", streamRec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func testAudiobookDetail() *catalog.ItemDetail {
|
||||
return &catalog.ItemDetail{
|
||||
ContentID: "book-1",
|
||||
Type: "audiobook",
|
||||
Title: "Book One",
|
||||
Audiobook: &catalog.AudiobookDetailExtension{
|
||||
Authors: []catalog.AudiobookPerson{{Name: "Author One"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func testAudioFile(id int, name string, partIndex int, duration int) *models.MediaFile {
|
||||
return &models.MediaFile{
|
||||
ID: id,
|
||||
ContentID: "book-1",
|
||||
FilePath: "/media/" + name,
|
||||
CodecAudio: "mp3",
|
||||
Container: "mp3",
|
||||
Duration: duration,
|
||||
PresentationPartIndex: partIndex,
|
||||
Chapters: []models.MediaChapter{{
|
||||
Index: 0,
|
||||
Title: name,
|
||||
StartSeconds: 0,
|
||||
EndSeconds: float64(duration),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func withAudioRouteParam(req *http.Request, key, value string) *http.Request {
|
||||
routeCtx := chi.RouteContext(req.Context())
|
||||
if routeCtx == nil {
|
||||
routeCtx = chi.NewRouteContext()
|
||||
}
|
||||
routeCtx.URLParams.Add(key, value)
|
||||
return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
|
||||
}
|
||||
@@ -0,0 +1,532 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// HandleGetAudiobook serves GET /api/v1/audiobooks/{id}. Returns the
|
||||
// audiobook media_item, its media_files (with embedded chapters),
|
||||
// author/narrator from item_people (kinds 7/8), and the caller's
|
||||
// per-profile listening progress.
|
||||
func (h *AudiobookHandler) HandleGetAudiobook(w http.ResponseWriter, r *http.Request) {
|
||||
contentID := chi.URLParam(r, "id")
|
||||
if contentID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "id is required")
|
||||
return
|
||||
}
|
||||
|
||||
item, err := h.Items.GetByID(r.Context(), contentID)
|
||||
if err != nil {
|
||||
if errors.Is(err, catalog.ErrItemNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load audiobook failed")
|
||||
return
|
||||
}
|
||||
if item == nil || item.Type != "audiobook" {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByContentID(r.Context(), contentID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load files failed")
|
||||
return
|
||||
}
|
||||
|
||||
author, narrator, err := h.fetchAuthorNarrator(r, contentID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load people failed")
|
||||
return
|
||||
}
|
||||
|
||||
progress := h.fetchProgress(r, contentID)
|
||||
|
||||
alsoByAuthor := h.fetchAlsoByAuthor(r.Context(), contentID)
|
||||
inSeries := h.fetchInSeries(r.Context(), contentID)
|
||||
similar := h.fetchSimilar(r.Context(), contentID)
|
||||
otherNarrations := h.fetchOtherNarrations(r.Context(), contentID)
|
||||
|
||||
resp := audiobookDetailResponse{
|
||||
Audiobook: audiobookDetailItem{
|
||||
ContentID: item.ContentID,
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterURL: h.presignPoster(r.Context(), item.PosterPath),
|
||||
Publisher: pickFirstString(item.Studios),
|
||||
Genres: append([]string(nil), item.Genres...),
|
||||
},
|
||||
Author: author,
|
||||
Narrator: narrator,
|
||||
Files: audiobookDetailFiles(files),
|
||||
Progress: progress,
|
||||
AlsoByAuthor: alsoByAuthor,
|
||||
InSeries: inSeries,
|
||||
SimilarAudiobooks: similar,
|
||||
OtherNarrations: otherNarrations,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// fetchAuthorNarrator retrieves the first author (kind=7) and narrator (kind=8)
|
||||
// name from item_people for the given content ID.
|
||||
func (h *AudiobookHandler) fetchAuthorNarrator(r *http.Request, contentID string) (author, narrator string, err error) {
|
||||
people, err := h.Items.GetPeople(r.Context(), contentID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, p := range people {
|
||||
switch p.Kind {
|
||||
case models.PersonKindAuthor:
|
||||
if author == "" {
|
||||
author = p.Name
|
||||
}
|
||||
case models.PersonKindNarrator:
|
||||
if narrator == "" {
|
||||
narrator = p.Name
|
||||
}
|
||||
}
|
||||
if author != "" && narrator != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
return author, narrator, nil
|
||||
}
|
||||
|
||||
// fetchAlsoByAuthor returns other audiobooks sharing any author (item_people
|
||||
// kind=PersonKindAuthor) with the given book. Capped at 12 to keep the rail
|
||||
// snappy on the client. Returns an empty slice when there is no author or
|
||||
// no other matching titles.
|
||||
func (h *AudiobookHandler) fetchAlsoByAuthor(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
const q = `
|
||||
SELECT m2.content_id, m2.title, COALESCE(m2.year, 0), COALESCE(m2.poster_path, '')
|
||||
FROM item_people ip1
|
||||
JOIN item_people ip2
|
||||
ON ip2.person_id = ip1.person_id
|
||||
AND ip2.kind = ip1.kind
|
||||
AND ip2.content_id <> ip1.content_id
|
||||
JOIN media_items m2
|
||||
ON m2.content_id = ip2.content_id
|
||||
AND m2.type = 'audiobook'
|
||||
WHERE ip1.content_id = $1
|
||||
AND ip1.kind = $2
|
||||
ORDER BY m2.year DESC NULLS LAST, LOWER(m2.sort_title)
|
||||
LIMIT 12
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID, models.PersonKindAuthor)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]audiobookRelatedItem, 0, 12)
|
||||
seen := make(map[string]struct{})
|
||||
for rows.Next() {
|
||||
var it audiobookRelatedItem
|
||||
var poster string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster); err != nil {
|
||||
return nil
|
||||
}
|
||||
if _, dup := seen[it.ContentID]; dup {
|
||||
continue
|
||||
}
|
||||
seen[it.ContentID] = struct{}{}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
out = append(out, it)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchSimilar returns the "You might also like" rail. Prefers embedding-based
|
||||
// nearest-neighbor search when the source book has an embedding; falls back to
|
||||
// shared-genre matching otherwise (book is brand new, embedding job hasn't run,
|
||||
// or the embedding lookup errored). The genre fallback excludes same-author
|
||||
// books so the rail complements rather than duplicates also_by_author.
|
||||
func (h *AudiobookHandler) fetchSimilar(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Recs != nil {
|
||||
if items := h.fetchSimilarByEmbedding(ctx, contentID); items != nil {
|
||||
return items
|
||||
}
|
||||
}
|
||||
return h.fetchSimilarByGenres(ctx, contentID)
|
||||
}
|
||||
|
||||
// fetchSimilarByEmbedding returns nearest-neighbor audiobooks by cosine
|
||||
// similarity on the gemini embedding vector. Returns nil (not an empty slice)
|
||||
// when the source book has no embedding yet, so the caller can fall back to
|
||||
// the genre-based recommender.
|
||||
func (h *AudiobookHandler) fetchSimilarByEmbedding(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Recs == nil || h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
emb, err := h.Recs.GetEmbedding(ctx, contentID)
|
||||
if err != nil || emb == nil {
|
||||
return nil
|
||||
}
|
||||
// Pull extra candidates so we can drop the source book's own author from
|
||||
// the rail (they belong in also_by_author).
|
||||
scored, err := h.Recs.FindSimilar(ctx, emb, []string{contentID}, "audiobook", 36)
|
||||
if err != nil || len(scored) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]string, 0, len(scored))
|
||||
for _, s := range scored {
|
||||
ids = append(ids, s.MediaItemID)
|
||||
}
|
||||
const q = `
|
||||
WITH this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $1 AND kind = $2
|
||||
)
|
||||
SELECT m.content_id, m.title, COALESCE(m.year, 0), COALESCE(m.poster_path, '')
|
||||
FROM media_items m
|
||||
WHERE m.content_id = ANY($3)
|
||||
AND m.type = 'audiobook'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $2
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID, models.PersonKindAuthor, ids)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
byID := make(map[string]audiobookRelatedItem, len(scored))
|
||||
for rows.Next() {
|
||||
var it audiobookRelatedItem
|
||||
var poster string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster); err != nil {
|
||||
return nil
|
||||
}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
byID[it.ContentID] = it
|
||||
}
|
||||
out := make([]audiobookRelatedItem, 0, 12)
|
||||
for _, s := range scored {
|
||||
if it, ok := byID[s.MediaItemID]; ok {
|
||||
out = append(out, it)
|
||||
if len(out) >= 12 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchOtherNarrations returns sibling audiobook items that share the same
|
||||
// "core" title (after stripping the common "read by X" / "(Read by X)" /
|
||||
// "(UK Version: Read by X)" suffixes) and the same author. Powers the
|
||||
// narrator picker dropdown on the detail page when a single book has
|
||||
// multiple narration editions stored as separate items in the catalog.
|
||||
func (h *AudiobookHandler) fetchOtherNarrations(ctx context.Context, contentID string) []audiobookNarration {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
// The regex strips a trailing block of the form "[ separator] [( ]read by ...[) ]".
|
||||
// Tested against patterns in the corpus: "Foo Read By X",
|
||||
// "Foo - read by X", "Foo (Read by X)", "Foo (UK Version: Read by X)".
|
||||
const stripRE = `\s*\(?\s*[-:,]?\s*(UK Version:?|US Version:?)?\s*[Rr]ead [Bb]y [^()]+\)?\s*$`
|
||||
const q = `
|
||||
WITH this_book AS (
|
||||
SELECT trim(regexp_replace(title, $1, '', 'g')) AS norm_title
|
||||
FROM media_items WHERE content_id = $2
|
||||
),
|
||||
this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $2 AND kind = $3
|
||||
)
|
||||
SELECT
|
||||
m.content_id,
|
||||
m.title,
|
||||
COALESCE(MAX(p.name), '') AS narrator,
|
||||
COALESCE(m.year, 0)
|
||||
FROM media_items m
|
||||
LEFT JOIN item_people ipn ON ipn.content_id = m.content_id AND ipn.kind = $4
|
||||
LEFT JOIN people p ON p.id = ipn.person_id
|
||||
WHERE m.type = 'audiobook'
|
||||
AND m.content_id <> $2
|
||||
AND trim(regexp_replace(m.title, $1, '', 'g')) = (SELECT norm_title FROM this_book)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $3
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)
|
||||
GROUP BY m.content_id, m.title, m.year
|
||||
ORDER BY m.year DESC NULLS LAST, m.title
|
||||
LIMIT 8
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q,
|
||||
stripRE, contentID, models.PersonKindAuthor, models.PersonKindNarrator)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]audiobookNarration, 0, 4)
|
||||
for rows.Next() {
|
||||
var n audiobookNarration
|
||||
if err := rows.Scan(&n.ContentID, &n.Title, &n.Narrator, &n.Year); err != nil {
|
||||
return nil
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchSimilarByGenres returns up to 12 audiobooks sharing genres with the
|
||||
// given book, excluding books by the same author so the rail complements
|
||||
// rather than duplicates also_by_author. Used as a fallback when no
|
||||
// embedding exists for the source yet.
|
||||
func (h *AudiobookHandler) fetchSimilarByGenres(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
const q = `
|
||||
WITH this_genres AS (
|
||||
SELECT unnest(genres) AS g FROM media_items WHERE content_id = $1
|
||||
),
|
||||
this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $1 AND kind = $2
|
||||
)
|
||||
SELECT m.content_id, m.title, COALESCE(m.year, 0), COALESCE(m.poster_path, '')
|
||||
FROM media_items m
|
||||
WHERE m.type = 'audiobook'
|
||||
AND m.content_id <> $1
|
||||
AND m.genres && (SELECT array_agg(g) FROM this_genres)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $2
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)
|
||||
ORDER BY
|
||||
cardinality(ARRAY(SELECT unnest(m.genres) INTERSECT SELECT g FROM this_genres)) DESC,
|
||||
COALESCE(m.year, 0) DESC,
|
||||
LOWER(m.sort_title)
|
||||
LIMIT 12
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID, models.PersonKindAuthor)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]audiobookRelatedItem, 0, 12)
|
||||
for rows.Next() {
|
||||
var it audiobookRelatedItem
|
||||
var poster string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster); err != nil {
|
||||
return nil
|
||||
}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
out = append(out, it)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchInSeries returns the series the audiobook belongs to (if any) along
|
||||
// with the ordered list of sibling entries. Driven by the audiobook_series
|
||||
// table populated by either the scanner (tag-derived) or migration 145's
|
||||
// best-effort title-pattern backfill.
|
||||
func (h *AudiobookHandler) fetchInSeries(ctx context.Context, contentID string) *audiobookSeriesGroup {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
const q = `
|
||||
SELECT
|
||||
m.content_id,
|
||||
m.title,
|
||||
COALESCE(m.year, 0),
|
||||
COALESCE(m.poster_path, ''),
|
||||
s.series_name,
|
||||
s.series_index
|
||||
FROM audiobook_series root
|
||||
JOIN audiobook_series s ON LOWER(s.series_name) = LOWER(root.series_name)
|
||||
JOIN media_items m ON m.content_id = s.content_id AND m.type = 'audiobook'
|
||||
WHERE root.content_id = $1
|
||||
ORDER BY s.series_index NULLS LAST, LOWER(m.sort_title)
|
||||
LIMIT 30
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var seriesName string
|
||||
entries := make([]audiobookRelatedItem, 0, 16)
|
||||
for rows.Next() {
|
||||
var (
|
||||
it audiobookRelatedItem
|
||||
poster string
|
||||
thisName string
|
||||
idxNum *float64
|
||||
)
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster, &thisName, &idxNum); err != nil {
|
||||
return nil
|
||||
}
|
||||
if seriesName == "" {
|
||||
seriesName = thisName
|
||||
}
|
||||
if idxNum != nil {
|
||||
n := int(*idxNum)
|
||||
if float64(n) == *idxNum {
|
||||
it.SeriesIndex = &n
|
||||
}
|
||||
}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
entries = append(entries, it)
|
||||
}
|
||||
// A series with only this book is not useful as a rail.
|
||||
if len(entries) < 2 {
|
||||
return nil
|
||||
}
|
||||
return &audiobookSeriesGroup{Name: seriesName, Entries: entries}
|
||||
}
|
||||
|
||||
// fetchProgress returns the caller's listening progress for this audiobook, or
|
||||
// nil when no progress exists or the caller is not authenticated with a profile.
|
||||
func (h *AudiobookHandler) fetchProgress(r *http.Request, contentID string) *audiobookDetailProgress {
|
||||
if h.StoreProvider == nil {
|
||||
return nil
|
||||
}
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
if userID == 0 {
|
||||
return nil
|
||||
}
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
if profileID == "" {
|
||||
profileID = r.Header.Get("X-Profile-Id")
|
||||
}
|
||||
if profileID == "" {
|
||||
return nil
|
||||
}
|
||||
store, err := h.StoreProvider.ForUser(r.Context(), userID)
|
||||
if err != nil || store == nil {
|
||||
return nil
|
||||
}
|
||||
wp, err := store.GetProgress(r.Context(), profileID, contentID)
|
||||
if err != nil || wp == nil {
|
||||
return nil
|
||||
}
|
||||
return &audiobookDetailProgress{
|
||||
PositionSeconds: wp.PositionSeconds,
|
||||
Completed: wp.Completed,
|
||||
UpdatedAt: wp.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
type audiobookDetailResponse struct {
|
||||
Audiobook audiobookDetailItem `json:"audiobook"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Narrator string `json:"narrator,omitempty"`
|
||||
Files []audiobookDetailFile `json:"files"`
|
||||
Progress *audiobookDetailProgress `json:"progress,omitempty"`
|
||||
AlsoByAuthor []audiobookRelatedItem `json:"also_by_author,omitempty"`
|
||||
InSeries *audiobookSeriesGroup `json:"in_series,omitempty"`
|
||||
SimilarAudiobooks []audiobookRelatedItem `json:"similar_audiobooks,omitempty"`
|
||||
OtherNarrations []audiobookNarration `json:"other_narrations,omitempty"`
|
||||
}
|
||||
|
||||
// audiobookNarration is a sibling narration (same book, different
|
||||
// narrator/edition). Returned alongside the primary item so the UI can
|
||||
// surface a narrator picker.
|
||||
type audiobookNarration struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Narrator string `json:"narrator,omitempty"`
|
||||
Year int `json:"year,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookRelatedItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
SeriesIndex *int `json:"series_index,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookSeriesGroup struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Entries []audiobookRelatedItem `json:"entries"`
|
||||
}
|
||||
|
||||
type audiobookDetailItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
Publisher string `json:"publisher,omitempty"`
|
||||
Genres []string `json:"genres,omitempty"`
|
||||
}
|
||||
|
||||
func (h *AudiobookHandler) presignPoster(ctx context.Context, path string) string {
|
||||
if path == "" || h.Detail == nil {
|
||||
return path
|
||||
}
|
||||
return h.Detail.PresignURL(ctx, path, "featured")
|
||||
}
|
||||
|
||||
func pickFirstString(values []string) string {
|
||||
for _, v := range values {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type audiobookDetailFile struct {
|
||||
ID int `json:"id"`
|
||||
Path string `json:"path"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
AudioChannels int `json:"audio_channels,omitempty"`
|
||||
Chapters []models.MediaChapter `json:"chapters,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookDetailProgress struct {
|
||||
PositionSeconds float64 `json:"position_seconds"`
|
||||
Completed bool `json:"completed"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func audiobookDetailFiles(files []*models.MediaFile) []audiobookDetailFile {
|
||||
out := make([]audiobookDetailFile, 0, len(files))
|
||||
for _, f := range files {
|
||||
out = append(out, audiobookDetailFile{
|
||||
ID: f.ID,
|
||||
Path: f.FilePath,
|
||||
DurationSeconds: f.Duration,
|
||||
Container: f.Container,
|
||||
CodecAudio: f.CodecAudio,
|
||||
Bitrate: f.Bitrate,
|
||||
AudioChannels: f.AudioChannels,
|
||||
Chapters: f.Chapters,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
// AudiobookHandler serves silo-native audiobook endpoints under
|
||||
// /api/v1/audiobooks/*. Sub-plan 3 implements list/detail/progress;
|
||||
// sub-plan 4's ABS-compat layer is separate.
|
||||
type AudiobookHandler struct {
|
||||
Items *catalog.ItemRepository
|
||||
Files *scanner.FileRepository
|
||||
StoreProvider userstore.UserStoreProvider
|
||||
Detail *catalog.DetailService
|
||||
// Recs is optional. When set, the detail handler uses embedding-based
|
||||
// nearest-neighbor search for the "You might also like" rail and falls
|
||||
// back to shared-genre matching only when no embedding exists for the
|
||||
// source book yet.
|
||||
Recs *recommendations.Repo
|
||||
}
|
||||
|
||||
// HandleListAudiobooks serves GET /api/v1/audiobooks?limit=&offset=.
|
||||
// Paginated catalog of media_items with type='audiobook'.
|
||||
func (h *AudiobookHandler) HandleListAudiobooks(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
genre := r.URL.Query().Get("genre")
|
||||
|
||||
pool := h.Files.Pool()
|
||||
ctx := r.Context()
|
||||
var total int
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM media_items
|
||||
WHERE type='audiobook' AND ($1 = '' OR $1 = ANY(genres))
|
||||
`, genre).Scan(&total); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks count failed")
|
||||
return
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT
|
||||
mi.content_id,
|
||||
mi.title,
|
||||
COALESCE(mi.year, 0),
|
||||
COALESCE(mi.poster_path, ''),
|
||||
COALESCE(SUM(mf.duration), 0)::int AS duration_seconds,
|
||||
COALESCE(MIN(mf.container), '') AS container,
|
||||
COALESCE(MIN(mf.codec_audio), '') AS codec_audio,
|
||||
COUNT(mf.id) AS file_count
|
||||
FROM media_items mi
|
||||
LEFT JOIN media_files mf ON mf.content_id = mi.content_id
|
||||
WHERE mi.type='audiobook'
|
||||
AND ($3 = '' OR $3 = ANY(mi.genres))
|
||||
GROUP BY mi.content_id, mi.title, mi.year, mi.poster_path, mi.sort_title
|
||||
ORDER BY LOWER(mi.sort_title), LOWER(mi.title)
|
||||
LIMIT $1 OFFSET $2
|
||||
`, limit, offset, genre)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks failed")
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]audiobookListItem, 0, limit)
|
||||
for rows.Next() {
|
||||
var it audiobookListItem
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &it.PosterURL, &it.DurationSeconds, &it.Container, &it.CodecAudio, &it.FileCount); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "scan audiobook row failed")
|
||||
return
|
||||
}
|
||||
if h.Detail != nil && it.PosterURL != "" {
|
||||
it.PosterURL = h.Detail.PresignURL(ctx, it.PosterURL, "card")
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "iterate audiobook rows failed")
|
||||
return
|
||||
}
|
||||
resp := struct {
|
||||
Items []audiobookListItem `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}{Items: items, Total: total, Limit: limit, Offset: offset}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
type audiobookListItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
FileCount int `json:"file_count"`
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
type audiobookProgressRequest struct {
|
||||
PositionSeconds float64 `json:"position_seconds"`
|
||||
MediaFileID int `json:"media_file_id,omitempty"`
|
||||
}
|
||||
|
||||
// HandleReportAudiobookProgress serves POST /api/v1/audiobooks/{id}/progress.
|
||||
// UPSERTs into user_watch_progress for the caller's (user_id, profile_id, content_id).
|
||||
// Duration is unknown at the audiobook level (it lives on the media_file), so we
|
||||
// pass 0 and let the store compute completion via thresholds against position only.
|
||||
func (h *AudiobookHandler) HandleReportAudiobookProgress(w http.ResponseWriter, r *http.Request) {
|
||||
contentID := chi.URLParam(r, "id")
|
||||
if contentID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "id is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req audiobookProgressRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "invalid request body")
|
||||
return
|
||||
}
|
||||
if req.PositionSeconds < 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "position_seconds must be >= 0")
|
||||
return
|
||||
}
|
||||
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
if profileID == "" {
|
||||
profileID = r.Header.Get("X-Profile-Id")
|
||||
}
|
||||
if userID == 0 || profileID == "" {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "authentication required")
|
||||
return
|
||||
}
|
||||
|
||||
if h.StoreProvider == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "store unavailable")
|
||||
return
|
||||
}
|
||||
store, err := h.StoreProvider.ForUser(r.Context(), userID)
|
||||
if err != nil || store == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "store unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
// Duration is not carried in the progress body; pass 0 so the store
|
||||
// uses position-only heuristics. ProgressThresholds zero value means
|
||||
// "use defaults" (90% watched, 5% min-resume).
|
||||
if err := store.SetProgress(r.Context(), profileID, contentID, req.PositionSeconds, 0, userstore.ProgressThresholds{}); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "save progress failed")
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -36,7 +36,6 @@ type autoscanStore interface {
|
||||
CountAutoscanScans(ctx context.Context, filter autoscan.ScanListFilter) (int, error)
|
||||
ListEvents(ctx context.Context, filter autoscan.EventListFilter) ([]autoscan.EventWithRuns, error)
|
||||
CountEvents(ctx context.Context, filter autoscan.EventListFilter) (int, error)
|
||||
ListRunningEvents(ctx context.Context) ([]autoscan.Event, error)
|
||||
GetQueueSummary(ctx context.Context) (autoscan.QueueSummary, error)
|
||||
LatestEventAt(ctx context.Context) (*time.Time, error)
|
||||
}
|
||||
@@ -310,7 +309,7 @@ func (h *AutoscanHandler) HandleDeleteConnection(w http.ResponseWriter, r *http.
|
||||
// credentials: the connection link is by id only.
|
||||
type autoscanSourceResponse struct {
|
||||
ID string `json:"id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
ConnectionID *string `json:"connection_id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
@@ -332,7 +331,7 @@ func sourceResponse(s autoscan.Source) autoscanSourceResponse {
|
||||
config := normalizeSourceConfig(s.SourceConfig)
|
||||
return autoscanSourceResponse{
|
||||
ID: s.ID,
|
||||
PluginID: s.PluginID,
|
||||
InstallationID: s.InstallationID,
|
||||
CapabilityID: s.CapabilityID,
|
||||
ConnectionID: s.ConnectionID,
|
||||
Enabled: s.Enabled,
|
||||
@@ -365,9 +364,10 @@ func (h *AutoscanHandler) HandleListSources(w http.ResponseWriter, r *http.Reque
|
||||
// --- Available scan-source plugins (Add-source picker) ---
|
||||
|
||||
type autoscanScanSourcePluginResponse struct {
|
||||
PluginID string `json:"plugin_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
// HandleListAvailableScanSources returns every installed scan_source capability
|
||||
@@ -381,9 +381,10 @@ func (h *AutoscanHandler) HandleListAvailableScanSources(w http.ResponseWriter,
|
||||
out := make([]autoscanScanSourcePluginResponse, 0, len(available))
|
||||
for _, a := range available {
|
||||
out = append(out, autoscanScanSourcePluginResponse{
|
||||
PluginID: a.PluginID,
|
||||
CapabilityID: a.CapabilityID,
|
||||
DisplayName: a.DisplayName,
|
||||
InstallationID: a.InstallationID,
|
||||
CapabilityID: a.CapabilityID,
|
||||
PluginID: a.PluginID,
|
||||
DisplayName: a.DisplayName,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, struct {
|
||||
@@ -393,12 +394,12 @@ func (h *AutoscanHandler) HandleListAvailableScanSources(w http.ResponseWriter,
|
||||
|
||||
// --- Create source ---
|
||||
|
||||
// autoscanCreateSourceInput is the create payload: it names the scan_source
|
||||
// capability to bind (plugin_id + capability_id) plus the initial
|
||||
// binding/scheduling fields. Unlike the update payload, identity is supplied
|
||||
// here rather than read from an existing row.
|
||||
// autoscanCreateSourceInput is the create payload: it names the installed
|
||||
// scan_source capability to bind (installation_id + capability_id) plus the
|
||||
// initial binding/scheduling fields. Unlike the update payload, identity is
|
||||
// supplied here rather than read from an existing row.
|
||||
type autoscanCreateSourceInput struct {
|
||||
PluginID string `json:"plugin_id"`
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
ConnectionID *string `json:"connection_id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
@@ -419,11 +420,6 @@ func (h *AutoscanHandler) HandleCreateSource(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
capID := strings.TrimSpace(in.CapabilityID)
|
||||
pluginID := strings.TrimSpace(in.PluginID)
|
||||
if pluginID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "plugin_id is required")
|
||||
return
|
||||
}
|
||||
if capID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "capability_id is required")
|
||||
return
|
||||
@@ -436,7 +432,7 @@ func (h *AutoscanHandler) HandleCreateSource(w http.ResponseWriter, r *http.Requ
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
// Validate the (plugin_id, capability_id) is a currently-installed
|
||||
// Validate the (installation_id, capability_id) is a currently-installed
|
||||
// scan_source capability — a source may only be created against an installed
|
||||
// plugin capability.
|
||||
available, err := h.svc.ListAvailableScanSources(r.Context())
|
||||
@@ -444,13 +440,13 @@ func (h *AutoscanHandler) HandleCreateSource(w http.ResponseWriter, r *http.Requ
|
||||
writeAutoscanError(w, err)
|
||||
return
|
||||
}
|
||||
if !scanSourceInstalled(available, pluginID, capID) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "plugin_id + capability_id is not a currently-installed scan_source capability")
|
||||
if !scanSourceInstalled(available, in.InstallationID, capID) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "installation_id + capability_id is not a currently-installed scan_source capability")
|
||||
return
|
||||
}
|
||||
connArg := normalizeConnectionID(in.ConnectionID)
|
||||
created, err := h.repo.CreateSource(r.Context(), autoscan.Source{
|
||||
PluginID: pluginID,
|
||||
InstallationID: in.InstallationID,
|
||||
CapabilityID: capID,
|
||||
ConnectionID: connArg,
|
||||
Enabled: in.Enabled,
|
||||
@@ -466,18 +462,18 @@ func (h *AutoscanHandler) HandleCreateSource(w http.ResponseWriter, r *http.Requ
|
||||
writeJSON(w, http.StatusCreated, sourceResponse(created))
|
||||
}
|
||||
|
||||
// scanSourceInstalled reports whether (pluginID, capabilityID) is in the
|
||||
// scanSourceInstalled reports whether (installationID, capabilityID) is in the
|
||||
// set of currently-installed scan_source capabilities.
|
||||
func scanSourceInstalled(available []autoscan.AvailableScanSource, pluginID, capabilityID string) bool {
|
||||
func scanSourceInstalled(available []autoscan.AvailableScanSource, installationID int, capabilityID string) bool {
|
||||
for _, a := range available {
|
||||
if a.PluginID == pluginID && a.CapabilityID == capabilityID {
|
||||
if a.InstallationID == installationID && a.CapabilityID == capabilityID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// autoscanSourceInput is the source write payload. The (plugin_id,
|
||||
// autoscanSourceInput is the source write payload. The (installation_id,
|
||||
// capability_id) identity is read from the existing source row, so only the
|
||||
// schedulable/binding fields are accepted.
|
||||
//
|
||||
@@ -552,7 +548,7 @@ func (h *AutoscanHandler) HandleUpdateSource(w http.ResponseWriter, r *http.Requ
|
||||
// connection_id is a full-state field: nil means unbind, a UUID string
|
||||
// means bind. A whitespace-only string is normalised to nil (unbound).
|
||||
connArg := normalizeConnectionID(in.ConnectionID)
|
||||
// Update by id; identity (plugin_id, capability_id) is immutable and
|
||||
// Update by id; identity (installation_id, capability_id) is immutable and
|
||||
// preserved by the repo. A missing source maps to 404.
|
||||
updated, err := h.repo.UpdateSource(r.Context(), autoscan.Source{
|
||||
ID: id,
|
||||
@@ -585,8 +581,9 @@ func normalizeConnectionID(in *string) *string {
|
||||
}
|
||||
|
||||
// HandleDeleteSource removes a source row. This is the operator's escape hatch
|
||||
// for an orphaned source (one whose scan_source plugin was uninstalled or no
|
||||
// longer resolves cleanly). An unknown id maps to 404 via autoscan.ErrNotFound.
|
||||
// for an orphaned source (one whose scan_source plugin was uninstalled): the
|
||||
// poll loop skips such rows quietly, and this endpoint clears them. An unknown
|
||||
// id maps to 404 via autoscan.ErrNotFound.
|
||||
func (h *AutoscanHandler) HandleDeleteSource(w http.ResponseWriter, r *http.Request) {
|
||||
id := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if err := h.repo.DeleteSource(r.Context(), id); err != nil {
|
||||
@@ -713,7 +710,7 @@ type autoscanEventScanRunResponse struct {
|
||||
type autoscanEventResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
SourceID *string `json:"source_id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
@@ -756,7 +753,7 @@ func eventResponse(event autoscan.EventWithRuns) autoscanEventResponse {
|
||||
return autoscanEventResponse{
|
||||
ID: e.ID,
|
||||
SourceID: e.SourceID,
|
||||
PluginID: e.PluginID,
|
||||
InstallationID: e.InstallationID,
|
||||
CapabilityID: e.CapabilityID,
|
||||
StartedAt: e.StartedAt,
|
||||
CompletedAt: e.CompletedAt,
|
||||
@@ -797,9 +794,9 @@ func (h *AutoscanHandler) HandleListEvents(w http.ResponseWriter, r *http.Reques
|
||||
filter.Offset = offset
|
||||
}
|
||||
switch filter.Status {
|
||||
case "", autoscan.EventStatusRunning, autoscan.EventStatusSuccess, autoscan.EventStatusError, autoscan.EventStatusUnresolved:
|
||||
case "", autoscan.EventStatusSuccess, autoscan.EventStatusError, autoscan.EventStatusUnresolved:
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "status must be running, success, error, or unresolved")
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "status must be success, error, or unresolved")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -838,7 +835,7 @@ type autoscanScanResponse struct {
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
AutoscanEventID *int64 `json:"autoscan_event_id,omitempty"`
|
||||
SourceID *string `json:"source_id,omitempty"`
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
InstallationID *int `json:"installation_id,omitempty"`
|
||||
CapabilityID string `json:"capability_id,omitempty"`
|
||||
EventStatus string `json:"event_status,omitempty"`
|
||||
EventCompletedAt *time.Time `json:"event_completed_at,omitempty"`
|
||||
@@ -865,7 +862,7 @@ func autoscanScanRunResponse(scan autoscan.ScanWithEvent) autoscanScanResponse {
|
||||
CompletedAt: scan.CompletedAt,
|
||||
AutoscanEventID: scan.AutoscanEventID,
|
||||
SourceID: scan.SourceID,
|
||||
PluginID: scan.PluginID,
|
||||
InstallationID: scan.InstallationID,
|
||||
CapabilityID: scan.CapabilityID,
|
||||
EventStatus: string(scan.EventStatus),
|
||||
EventCompletedAt: scan.EventCompletedAt,
|
||||
@@ -926,35 +923,24 @@ func (h *AutoscanHandler) HandleListScans(w http.ResponseWriter, r *http.Request
|
||||
// --- Status ---
|
||||
|
||||
type autoscanStatusSource struct {
|
||||
ID string `json:"id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
ConnectionID *string `json:"connection_id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Label string `json:"label"`
|
||||
PathRewrites []autoscan.PathRewrite `json:"path_rewrites"`
|
||||
LastRunAt *time.Time `json:"last_run_at,omitempty"`
|
||||
LastError *string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
type autoscanRunningPollResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
SourceID *string `json:"source_id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
MarkerBefore *string `json:"marker_before,omitempty"`
|
||||
ID string `json:"id"`
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
ConnectionID *string `json:"connection_id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Label string `json:"label"`
|
||||
PathRewrites []autoscan.PathRewrite `json:"path_rewrites"`
|
||||
LastRunAt *time.Time `json:"last_run_at,omitempty"`
|
||||
LastError *string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
type autoscanStatusResponse struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Sources []autoscanStatusSource `json:"sources"`
|
||||
RunningPolls []autoscanRunningPollResponse `json:"running_polls"`
|
||||
ActiveScans int `json:"active_scans"`
|
||||
AcceptedScans int `json:"accepted_scans"`
|
||||
RunningScans int `json:"running_scans"`
|
||||
LatestEventAt *time.Time `json:"latest_event_at,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Sources []autoscanStatusSource `json:"sources"`
|
||||
ActiveScans int `json:"active_scans"`
|
||||
AcceptedScans int `json:"accepted_scans"`
|
||||
RunningScans int `json:"running_scans"`
|
||||
LatestEventAt *time.Time `json:"latest_event_at,omitempty"`
|
||||
}
|
||||
|
||||
func (h *AutoscanHandler) HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -974,11 +960,6 @@ func (h *AutoscanHandler) HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeAutoscanError(w, err)
|
||||
return
|
||||
}
|
||||
runningEvents, err := h.repo.ListRunningEvents(ctx)
|
||||
if err != nil {
|
||||
writeAutoscanError(w, err)
|
||||
return
|
||||
}
|
||||
latestEventAt, err := h.repo.LatestEventAt(ctx)
|
||||
if err != nil {
|
||||
writeAutoscanError(w, err)
|
||||
@@ -991,38 +972,20 @@ func (h *AutoscanHandler) HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
rewrites = []autoscan.PathRewrite{}
|
||||
}
|
||||
trimmed = append(trimmed, autoscanStatusSource{
|
||||
ID: src.ID,
|
||||
PluginID: src.PluginID,
|
||||
CapabilityID: src.CapabilityID,
|
||||
ConnectionID: src.ConnectionID,
|
||||
Enabled: src.Enabled,
|
||||
Label: src.Label,
|
||||
PathRewrites: rewrites,
|
||||
LastRunAt: src.LastRunAt,
|
||||
LastError: src.LastError,
|
||||
})
|
||||
}
|
||||
now := time.Now()
|
||||
runningPolls := make([]autoscanRunningPollResponse, 0, len(runningEvents))
|
||||
for _, event := range runningEvents {
|
||||
elapsed := now.Sub(event.StartedAt)
|
||||
if elapsed < 0 {
|
||||
elapsed = 0
|
||||
}
|
||||
runningPolls = append(runningPolls, autoscanRunningPollResponse{
|
||||
ID: event.ID,
|
||||
SourceID: event.SourceID,
|
||||
PluginID: event.PluginID,
|
||||
CapabilityID: event.CapabilityID,
|
||||
StartedAt: event.StartedAt,
|
||||
ElapsedMS: elapsed.Milliseconds(),
|
||||
MarkerBefore: event.MarkerBefore,
|
||||
ID: src.ID,
|
||||
InstallationID: src.InstallationID,
|
||||
CapabilityID: src.CapabilityID,
|
||||
ConnectionID: src.ConnectionID,
|
||||
Enabled: src.Enabled,
|
||||
Label: src.Label,
|
||||
PathRewrites: rewrites,
|
||||
LastRunAt: src.LastRunAt,
|
||||
LastError: src.LastError,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, autoscanStatusResponse{
|
||||
Enabled: s.Enabled,
|
||||
Sources: trimmed,
|
||||
RunningPolls: runningPolls,
|
||||
ActiveScans: queue.Active,
|
||||
AcceptedScans: queue.Accepted,
|
||||
RunningScans: queue.Running,
|
||||
|
||||
@@ -31,7 +31,6 @@ type fakeAutoscanStore struct {
|
||||
countScansFn func(autoscan.ScanListFilter) (int, error)
|
||||
listEventsFn func(autoscan.EventListFilter) ([]autoscan.EventWithRuns, error)
|
||||
countEventsFn func(autoscan.EventListFilter) (int, error)
|
||||
listRunningFn func() ([]autoscan.Event, error)
|
||||
queueSummaryFn func() (autoscan.QueueSummary, error)
|
||||
latestEventAtFn func() (*time.Time, error)
|
||||
}
|
||||
@@ -141,13 +140,6 @@ func (f *fakeAutoscanStore) CountEvents(_ context.Context, filter autoscan.Event
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (f *fakeAutoscanStore) ListRunningEvents(context.Context) ([]autoscan.Event, error) {
|
||||
if f.listRunningFn != nil {
|
||||
return f.listRunningFn()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeAutoscanStore) GetQueueSummary(context.Context) (autoscan.QueueSummary, error) {
|
||||
if f.queueSummaryFn != nil {
|
||||
return f.queueSummaryFn()
|
||||
@@ -281,7 +273,7 @@ func TestAutoscanHandleListSourcesListsOnly(t *testing.T) {
|
||||
store := &fakeAutoscanStore{
|
||||
listSourcesFn: func() ([]autoscan.Source, error) {
|
||||
listed = true
|
||||
return []autoscan.Source{{ID: "src-1", PluginID: "silo.autoscan.arr", CapabilityID: "arr"}}, nil
|
||||
return []autoscan.Source{{ID: "src-1", InstallationID: 1, CapabilityID: "arr"}}, nil
|
||||
},
|
||||
}
|
||||
h := NewAutoscanHandler(store, &fakeAutoscanTriggerer{})
|
||||
@@ -308,7 +300,7 @@ func TestAutoscanHandleListSourcesListsOnly(t *testing.T) {
|
||||
|
||||
func TestAutoscanHandleListAvailableScanSources(t *testing.T) {
|
||||
trig := &fakeAutoscanTriggerer{available: []autoscan.AvailableScanSource{
|
||||
{PluginID: "sonarr", CapabilityID: "arr", DisplayName: "Sonarr"},
|
||||
{InstallationID: 1, CapabilityID: "arr", PluginID: "sonarr", DisplayName: "Sonarr"},
|
||||
}}
|
||||
h := NewAutoscanHandler(&fakeAutoscanStore{}, trig)
|
||||
|
||||
@@ -339,19 +331,19 @@ func TestAutoscanHandleCreateSourceSucceeds(t *testing.T) {
|
||||
},
|
||||
}
|
||||
trig := &fakeAutoscanTriggerer{available: []autoscan.AvailableScanSource{
|
||||
{PluginID: "silo.autoscan.arr", CapabilityID: "arr"},
|
||||
{InstallationID: 1, CapabilityID: "arr"},
|
||||
}}
|
||||
h := NewAutoscanHandler(store, trig)
|
||||
|
||||
req := newAutoscanRequest("POST", "/api/v1/admin/autoscan/sources",
|
||||
`{"plugin_id":"silo.autoscan.arr","capability_id":"arr","connection_id":"conn-1","enabled":true}`, "")
|
||||
`{"installation_id":1,"capability_id":"arr","connection_id":"conn-1","enabled":true}`, "")
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleCreateSource(rec, req)
|
||||
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, want 201; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got.PluginID != "silo.autoscan.arr" || got.CapabilityID != "arr" || got.ConnectionID == nil || *got.ConnectionID != "conn-1" {
|
||||
if got.InstallationID != 1 || got.CapabilityID != "arr" || got.ConnectionID == nil || *got.ConnectionID != "conn-1" {
|
||||
t.Fatalf("unexpected created source: %+v", got)
|
||||
}
|
||||
var body autoscanSourceResponse
|
||||
@@ -371,14 +363,14 @@ func TestAutoscanHandleCreateSourceRejectsUnknownCapability(t *testing.T) {
|
||||
return s, nil
|
||||
},
|
||||
}
|
||||
// Installed set does NOT include ("silo.autoscan.arr", "arr").
|
||||
// Installed set does NOT include (1, "arr").
|
||||
trig := &fakeAutoscanTriggerer{available: []autoscan.AvailableScanSource{
|
||||
{PluginID: "silo.autoscan.other", CapabilityID: "other"},
|
||||
{InstallationID: 2, CapabilityID: "other"},
|
||||
}}
|
||||
h := NewAutoscanHandler(store, trig)
|
||||
|
||||
req := newAutoscanRequest("POST", "/api/v1/admin/autoscan/sources",
|
||||
`{"plugin_id":"silo.autoscan.arr","capability_id":"arr","enabled":false}`, "")
|
||||
`{"installation_id":1,"capability_id":"arr","enabled":false}`, "")
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleCreateSource(rec, req)
|
||||
|
||||
@@ -402,12 +394,12 @@ func TestAutoscanHandleCreateSourceEnableWithoutConnectionSucceeds(t *testing.T)
|
||||
},
|
||||
}
|
||||
trig := &fakeAutoscanTriggerer{available: []autoscan.AvailableScanSource{
|
||||
{PluginID: "silo.autoscan.arr", CapabilityID: "arr"},
|
||||
{InstallationID: 1, CapabilityID: "arr"},
|
||||
}}
|
||||
h := NewAutoscanHandler(store, trig)
|
||||
|
||||
req := newAutoscanRequest("POST", "/api/v1/admin/autoscan/sources",
|
||||
`{"plugin_id":"silo.autoscan.arr","capability_id":"arr","enabled":true}`, "")
|
||||
`{"installation_id":1,"capability_id":"arr","enabled":true}`, "")
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleCreateSource(rec, req)
|
||||
|
||||
@@ -623,7 +615,7 @@ func TestAutoscanHandleUpdateSourceEnableWithoutConnectionSucceeds(t *testing.T)
|
||||
var got autoscan.Source
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: nil}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: nil}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
got = s
|
||||
@@ -649,7 +641,7 @@ func TestAutoscanHandleUpdateSourceEnableWithConnectionSucceeds(t *testing.T) {
|
||||
var got autoscan.Source
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
got = s
|
||||
@@ -677,7 +669,7 @@ func TestAutoscanHandleUpdateSourceBindConnectionSucceeds(t *testing.T) {
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
// Source starts with no connection.
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: nil}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: nil}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
got = s
|
||||
@@ -705,7 +697,7 @@ func TestAutoscanHandleUpdateSourceUnbindConnectionSucceeds(t *testing.T) {
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
// Source starts with a connection already bound.
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
got = s
|
||||
@@ -735,7 +727,7 @@ func TestAutoscanHandleUpdateSourceUnbindWhileEnabledSucceeds(t *testing.T) {
|
||||
var got autoscan.Source
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
got = s
|
||||
@@ -933,7 +925,7 @@ func TestAutoscanHandleUpdateSourceRoundTripsPathRewrites(t *testing.T) {
|
||||
var got autoscan.Source
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
got = s
|
||||
@@ -998,7 +990,7 @@ func TestAutoscanHandleUpdateSourceRejectsBlankRewrite(t *testing.T) {
|
||||
upserted := false
|
||||
store := &fakeAutoscanStore{
|
||||
getSourceFn: func(id string) (autoscan.Source, error) {
|
||||
return autoscan.Source{ID: id, PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
return autoscan.Source{ID: id, InstallationID: 1, CapabilityID: "arr", ConnectionID: ptr("conn-1")}, nil
|
||||
},
|
||||
updateSourceFn: func(s autoscan.Source) (autoscan.Source, error) {
|
||||
upserted = true
|
||||
@@ -1028,7 +1020,7 @@ func TestAutoscanHandleListScansSerializesFiltersAndEventContext(t *testing.T) {
|
||||
eventCompleted := completed.Add(-1 * time.Second)
|
||||
eventID := int64(42)
|
||||
sourceID := "src-1"
|
||||
pluginID := "silo.autoscan.cephfs"
|
||||
installationID := 7
|
||||
var gotFilter autoscan.ScanListFilter
|
||||
var gotCountFilter autoscan.ScanListFilter
|
||||
store := &fakeAutoscanStore{
|
||||
@@ -1048,7 +1040,7 @@ func TestAutoscanHandleListScansSerializesFiltersAndEventContext(t *testing.T) {
|
||||
},
|
||||
AutoscanEventID: &eventID,
|
||||
SourceID: &sourceID,
|
||||
PluginID: pluginID,
|
||||
InstallationID: &installationID,
|
||||
CapabilityID: "cephfs",
|
||||
EventStatus: autoscan.EventStatusSuccess,
|
||||
EventCompletedAt: &eventCompleted,
|
||||
@@ -1092,7 +1084,7 @@ func TestAutoscanHandleListScansSerializesFiltersAndEventContext(t *testing.T) {
|
||||
if scan.AutoscanEventID == nil || *scan.AutoscanEventID != eventID || scan.EventStatus != "success" {
|
||||
t.Fatalf("event context = %+v", scan)
|
||||
}
|
||||
if scan.SourceID == nil || *scan.SourceID != sourceID || scan.PluginID != pluginID || scan.CapabilityID != "cephfs" {
|
||||
if scan.SourceID == nil || *scan.SourceID != sourceID || scan.InstallationID == nil || *scan.InstallationID != installationID || scan.CapabilityID != "cephfs" {
|
||||
t.Fatalf("source context = %+v", scan)
|
||||
}
|
||||
}
|
||||
@@ -1136,7 +1128,7 @@ func TestAutoscanHandleListEventsSerializesFiltersAndRuns(t *testing.T) {
|
||||
Event: autoscan.Event{
|
||||
ID: 42,
|
||||
SourceID: &sourceID,
|
||||
PluginID: "silo.autoscan.cephfs",
|
||||
InstallationID: 7,
|
||||
CapabilityID: "cephfs",
|
||||
StartedAt: started,
|
||||
CompletedAt: completed,
|
||||
@@ -1266,30 +1258,18 @@ func TestAutoscanHandleUpdateSourceNormalizesLabel(t *testing.T) {
|
||||
|
||||
func TestAutoscanHandleStatusReturnsTrimmedSources(t *testing.T) {
|
||||
latestEventAt := time.Date(2026, 6, 4, 14, 30, 0, 0, time.UTC)
|
||||
runningStartedAt := time.Now().Add(-90 * time.Second)
|
||||
sourceID := "src-1"
|
||||
store := &fakeAutoscanStore{
|
||||
getSettingsFn: func() (autoscan.Settings, error) {
|
||||
return autoscan.Settings{Enabled: true}, nil
|
||||
},
|
||||
listSourcesFn: func() ([]autoscan.Source, error) {
|
||||
return []autoscan.Source{
|
||||
{ID: "src-1", PluginID: "silo.autoscan.cephfs", CapabilityID: "scan_source", ConnectionID: ptr("conn-1"), Enabled: true},
|
||||
{ID: "src-1", InstallationID: 7, CapabilityID: "scan_source", ConnectionID: ptr("conn-1"), Enabled: true},
|
||||
}, nil
|
||||
},
|
||||
queueSummaryFn: func() (autoscan.QueueSummary, error) {
|
||||
return autoscan.QueueSummary{Active: 3, Accepted: 2, Running: 1}, nil
|
||||
},
|
||||
listRunningFn: func() ([]autoscan.Event, error) {
|
||||
return []autoscan.Event{{
|
||||
ID: 44,
|
||||
SourceID: &sourceID,
|
||||
PluginID: "silo.autoscan.cephfs",
|
||||
CapabilityID: "cephfs",
|
||||
StartedAt: runningStartedAt,
|
||||
Status: autoscan.EventStatusRunning,
|
||||
}}, nil
|
||||
},
|
||||
latestEventAtFn: func() (*time.Time, error) {
|
||||
return &latestEventAt, nil
|
||||
},
|
||||
@@ -1316,16 +1296,6 @@ func TestAutoscanHandleStatusReturnsTrimmedSources(t *testing.T) {
|
||||
if body.ActiveScans != 3 || body.AcceptedScans != 2 || body.RunningScans != 1 {
|
||||
t.Errorf("queue summary = %+v", body)
|
||||
}
|
||||
if len(body.RunningPolls) != 1 {
|
||||
t.Fatalf("running_polls = %+v", body.RunningPolls)
|
||||
}
|
||||
poll := body.RunningPolls[0]
|
||||
if poll.ID != 44 || poll.SourceID == nil || *poll.SourceID != sourceID || poll.PluginID != "silo.autoscan.cephfs" || poll.CapabilityID != "cephfs" {
|
||||
t.Fatalf("running poll identity = %+v", poll)
|
||||
}
|
||||
if poll.ElapsedMS <= 0 {
|
||||
t.Fatalf("running poll elapsed_ms = %d, want > 0", poll.ElapsedMS)
|
||||
}
|
||||
if body.LatestEventAt == nil || !body.LatestEventAt.Equal(latestEventAt) {
|
||||
t.Errorf("latest_event_at = %v, want %v", body.LatestEventAt, latestEventAt)
|
||||
}
|
||||
|
||||
@@ -221,9 +221,6 @@ func (h *CatalogHandler) HandleGetCatalogFacetSearch(w http.ResponseWriter, r *h
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "limit must be a positive integer")
|
||||
return
|
||||
}
|
||||
if n > 100 {
|
||||
n = 100
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
|
||||
|
||||
@@ -497,7 +497,7 @@ func (h *CatalogResourceHandler) enrichItemDetail(r *http.Request, detail *catal
|
||||
detail.SeasonUserData = h.items.getAggregateUserData(r, episodes)
|
||||
}
|
||||
}
|
||||
case "movie", "episode", "audiobook":
|
||||
case "movie", "episode":
|
||||
detail.SeasonUserData = h.items.getLeafUserData(r, detail.ContentID)
|
||||
applyEffectiveEditionPreference(detail.SeasonUserData, &detail.EffectiveVersionEditionKey)
|
||||
}
|
||||
|
||||
@@ -391,17 +391,13 @@ func (h *EventsHandler) writeSnapshotFrame(
|
||||
) error {
|
||||
data, err := h.snapshotForChannel(r, claims, channel)
|
||||
if err != nil {
|
||||
slog.Error(
|
||||
"events: failed to build initial snapshot",
|
||||
"channel",
|
||||
channel,
|
||||
"user_id",
|
||||
claims.UserID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
|
||||
return nil
|
||||
_ = writeWebSocketControl(
|
||||
conn,
|
||||
websocket.CloseMessage,
|
||||
websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "snapshot failed"),
|
||||
)
|
||||
return err
|
||||
}
|
||||
return writeWebSocketJSON(conn, evt.EventsSnapshotMessage{
|
||||
Type: "snapshot",
|
||||
|
||||
@@ -150,7 +150,7 @@ func (h *ItemsHandler) maybeRequestStaleEpisodeMetadataRefresh(ctx context.Conte
|
||||
if h == nil || h.metadataRefreshRequester == nil || episode == nil {
|
||||
return
|
||||
}
|
||||
if metadata.EpisodeHasActionableMetadataDebt(episode, time.Now()) {
|
||||
if metadata.EpisodeHasIncompleteMetadata(episode, time.Now()) {
|
||||
h.requestStaleMetadataRefresh(ctx, "episode", episode.ContentID)
|
||||
}
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func (h *ItemsHandler) maybeRequestStaleSeasonMetadataRefresh(ctx context.Contex
|
||||
}
|
||||
now := time.Now()
|
||||
for _, episode := range episodes {
|
||||
if metadata.EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
if metadata.EpisodeHasIncompleteMetadata(episode, now) {
|
||||
h.requestStaleMetadataRefresh(ctx, "season", seasonID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2036,10 +2036,8 @@ func (h *LibraryHandler) seedDefaultChain(ctx context.Context, libraryType strin
|
||||
levels = []string{"series", "season", "episode"}
|
||||
case "movies", "movie":
|
||||
levels = []string{"movie"}
|
||||
case "audiobooks", "audiobook":
|
||||
levels = []string{"audiobook"}
|
||||
case "mixed":
|
||||
levels = []string{"movie", "series", "season", "episode", "audiobook"}
|
||||
levels = []string{"movie", "series", "season", "episode"}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,849 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimw "github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
)
|
||||
|
||||
// MarkerFileResolver loads a media file by id for the manual-marker API.
|
||||
type MarkerFileResolver interface {
|
||||
GetByID(ctx context.Context, id int) (*models.MediaFile, error)
|
||||
GetByContentID(ctx context.Context, contentID string) ([]*models.MediaFile, error)
|
||||
GetByEpisodeID(ctx context.Context, episodeID string) ([]*models.MediaFile, error)
|
||||
}
|
||||
|
||||
// ManualMarkerWriter persists/clears manual markers.
|
||||
type ManualMarkerWriter interface {
|
||||
UpsertMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate) (bool, error)
|
||||
ClearMarkers(ctx context.Context, fileID int, segments []string) (bool, error)
|
||||
UpsertAndClearMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate, clearSegments []string) (bool, error)
|
||||
}
|
||||
|
||||
// MarkerContributor submits a file's eligible markers to enabled providers.
|
||||
type MarkerContributor interface {
|
||||
ContributeFile(ctx context.Context, file *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error)
|
||||
}
|
||||
|
||||
// MarkerContributionLister reads contribution history for a file.
|
||||
type MarkerContributionLister interface {
|
||||
ListByFile(ctx context.Context, fileID int) ([]markers.ContributionRow, error)
|
||||
}
|
||||
|
||||
type MarkerAuditLister interface {
|
||||
ListMarkerEditAudit(ctx context.Context, fileIDs []int, limit int) ([]scanner.MarkerEditAuditRow, error)
|
||||
ListAllMarkerEditAudit(ctx context.Context, limit int) ([]scanner.MarkerEditAuditRow, error)
|
||||
}
|
||||
|
||||
type MarkerPermissionUserLoader interface {
|
||||
GetByID(ctx context.Context, id int) (*models.User, error)
|
||||
}
|
||||
|
||||
// MarkersHandler serves the manual-marker + contribution API. The marker
|
||||
// read/write/clear routes are mounted for any authenticated viewer (users fix
|
||||
// and create markers from the player); the contribution + history routes stay
|
||||
// admin-only. A successful manual write fires a background contribution run
|
||||
// (see maybeContribute) so corrected markers reach enabled providers.
|
||||
type MarkersHandler struct {
|
||||
Files MarkerFileResolver
|
||||
Writer ManualMarkerWriter
|
||||
Contributor MarkerContributor
|
||||
Contributions MarkerContributionLister
|
||||
AuditHistory MarkerAuditLister
|
||||
Notifier PlaybackMarkerUpdateNotifier
|
||||
Users MarkerPermissionUserLoader
|
||||
// Authorizer enforces per-item access on file lookups so a viewer can only
|
||||
// edit markers for content they can actually watch. When nil (tests) the
|
||||
// handler falls back to an unchecked lookup.
|
||||
Authorizer *MediaFileAuthorizer
|
||||
// BaseContext is the lifetime context for detached background work (set from
|
||||
// the app context) so contribution goroutines cancel on shutdown.
|
||||
BaseContext context.Context
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewMarkersHandler constructs the handler.
|
||||
func NewMarkersHandler(files MarkerFileResolver, writer ManualMarkerWriter, contributor MarkerContributor, contributions MarkerContributionLister, notifier PlaybackMarkerUpdateNotifier, logger *slog.Logger) *MarkersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &MarkersHandler{Files: files, Writer: writer, Contributor: contributor, Contributions: contributions, Notifier: notifier, logger: logger}
|
||||
}
|
||||
|
||||
var manualMarkerConfidence = 1.0
|
||||
|
||||
const manualMarkerAlgorithm = "manual:v1"
|
||||
|
||||
// markerContributeTimeout bounds the detached contribution run kicked off after
|
||||
// a manual save so a slow or rate-limited provider can't leak goroutines.
|
||||
const markerContributeTimeout = 30 * time.Second
|
||||
|
||||
var markerSegmentNames = []string{"intro", "credits", "recap", "preview"}
|
||||
|
||||
type segmentInput struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
}
|
||||
|
||||
type segmentMarker struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
Source *string `json:"source"`
|
||||
Provider *string `json:"provider"`
|
||||
Confidence *float64 `json:"confidence"`
|
||||
Algorithm *string `json:"algorithm"`
|
||||
DetectedAt *time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
type fileMarkersResponse struct {
|
||||
FileID int `json:"file_id"`
|
||||
Intro segmentMarker `json:"intro"`
|
||||
Credits segmentMarker `json:"credits"`
|
||||
Recap segmentMarker `json:"recap"`
|
||||
Preview segmentMarker `json:"preview"`
|
||||
}
|
||||
|
||||
type contributionOutcomeResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
Segment string `json:"segment"`
|
||||
Status string `json:"status"`
|
||||
SubmissionID string `json:"submission_id,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
RetryAfterSeconds int `json:"retry_after_seconds,omitempty"`
|
||||
}
|
||||
|
||||
type contributionRowResponse struct {
|
||||
ID string `json:"id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
Provider string `json:"provider"`
|
||||
Segment string `json:"segment"`
|
||||
Source string `json:"source"`
|
||||
SubmittedStartMS *int64 `json:"submitted_start_ms,omitempty"`
|
||||
SubmittedEndMS *int64 `json:"submitted_end_ms,omitempty"`
|
||||
VideoDurationMS *int64 `json:"video_duration_ms,omitempty"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
SubmissionID *string `json:"submission_id,omitempty"`
|
||||
Status string `json:"status"`
|
||||
HTTPStatus *int `json:"http_status,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
SubmittedAt *time.Time `json:"submitted_at,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type markerEditAuditResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
ItemID *string `json:"item_id,omitempty"`
|
||||
ItemType *string `json:"item_type,omitempty"`
|
||||
MediaTitle *string `json:"media_title,omitempty"`
|
||||
FilePath *string `json:"file_path,omitempty"`
|
||||
Segment string `json:"segment"`
|
||||
Action string `json:"action"`
|
||||
Before *segmentMarker `json:"before"`
|
||||
After *segmentMarker `json:"after"`
|
||||
UserID *int `json:"user_id,omitempty"`
|
||||
Username *string `json:"username,omitempty"`
|
||||
ImpersonatorUserID *int `json:"impersonator_user_id,omitempty"`
|
||||
ImpersonatorUsername *string `json:"impersonator_username,omitempty"`
|
||||
APIKeyID *int64 `json:"api_key_id,omitempty"`
|
||||
RequestID *string `json:"request_id,omitempty"`
|
||||
ClientIP *string `json:"client_ip,omitempty"`
|
||||
UserAgent *string `json:"user_agent,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func fileMarkers(file *models.MediaFile) fileMarkersResponse {
|
||||
return fileMarkersResponse{
|
||||
FileID: file.ID,
|
||||
Intro: segmentMarker{file.IntroStart, file.IntroEnd, file.IntroMarkersSource, file.IntroMarkersProvider,
|
||||
file.IntroMarkersConfidence, file.IntroMarkersAlgorithm, file.IntroMarkersDetectedAt},
|
||||
Credits: segmentMarker{file.CreditsStart, file.CreditsEnd, file.CreditsMarkersSource, file.CreditsMarkersProvider,
|
||||
file.CreditsMarkersConfidence, file.CreditsMarkersAlgorithm, file.CreditsMarkersDetectedAt},
|
||||
Recap: segmentMarker{file.RecapStart, file.RecapEnd, file.RecapMarkersSource, file.RecapMarkersProvider,
|
||||
file.RecapMarkersConfidence, file.RecapMarkersAlgorithm, file.RecapMarkersDetectedAt},
|
||||
Preview: segmentMarker{file.PreviewStart, file.PreviewEnd, file.PreviewMarkersSource, file.PreviewMarkersProvider,
|
||||
file.PreviewMarkersConfidence, file.PreviewMarkersAlgorithm, file.PreviewMarkersDetectedAt},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) loadFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
|
||||
return nil, false
|
||||
}
|
||||
fileID, err := strconv.Atoi(chi.URLParam(r, "fileId"))
|
||||
if err != nil || fileID <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid file id is required")
|
||||
return nil, false
|
||||
}
|
||||
return h.authorizeFile(w, r, fileID)
|
||||
}
|
||||
|
||||
// authorizeFile loads a file, enforcing per-item access when an authorizer is
|
||||
// configured. The 404-on-denial mirrors the playback/subtitle paths so marker
|
||||
// editing can't be used to probe content outside a profile's library scope.
|
||||
func (h *MarkersHandler) authorizeFile(w http.ResponseWriter, r *http.Request, fileID int) (*models.MediaFile, bool) {
|
||||
if h.Authorizer != nil {
|
||||
file, err := h.Authorizer.Authorize(r, fileID)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, catalog.ErrItemNotFound), errors.Is(err, catalog.ErrEpisodeNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
default:
|
||||
h.logger.Error("markers: authorize failed", "file_id", fileID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
return file, true
|
||||
}
|
||||
file, err := h.Files.GetByID(r.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
return nil, false
|
||||
}
|
||||
return file, true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) loadItemPrimaryFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
|
||||
return nil, false
|
||||
}
|
||||
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if itemID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: episode file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return nil, false
|
||||
}
|
||||
if len(files) == 0 {
|
||||
files, err = h.Files.GetByContentID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: content file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
// Re-validate the resolved primary file through the authorizer so item-id
|
||||
// edits are access-checked the same way file-id edits are.
|
||||
if h.Authorizer != nil {
|
||||
for _, file := range files {
|
||||
if file == nil {
|
||||
continue
|
||||
}
|
||||
authorized, err := h.Authorizer.Authorize(r, file.ID)
|
||||
if err == nil {
|
||||
return authorized, true
|
||||
}
|
||||
if errors.Is(err, catalog.ErrItemNotFound) || errors.Is(err, catalog.ErrEpisodeNotFound) {
|
||||
continue
|
||||
}
|
||||
h.logger.Error("markers: authorize item file failed", "item_id", itemID, "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
return nil, false
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
for _, file := range files {
|
||||
if file != nil {
|
||||
return file, true
|
||||
}
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) requireMarkerEdit(w http.ResponseWriter, r *http.Request) bool {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
|
||||
return false
|
||||
}
|
||||
if claims.Role == "admin" {
|
||||
return true
|
||||
}
|
||||
if h == nil || h.Users == nil {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
|
||||
return false
|
||||
}
|
||||
user, err := h.Users.GetByID(r.Context(), claims.UserID)
|
||||
if err != nil || user == nil || !auth.HasEffectivePermission(user, auth.PermissionMarkerEdit) {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) auditContext(r *http.Request) context.Context {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
return r.Context()
|
||||
}
|
||||
audit := scanner.MarkerAuditContext{
|
||||
UserID: &claims.UserID,
|
||||
ImpersonatorUserID: claims.ImpersonatorUserID,
|
||||
RequestID: chimw.GetReqID(r.Context()),
|
||||
ClientIP: clientip.FromContext(r.Context()),
|
||||
UserAgent: r.UserAgent(),
|
||||
}
|
||||
if claims.APIKeyID > 0 {
|
||||
apiKeyID := claims.APIKeyID
|
||||
audit.APIKeyID = &apiKeyID
|
||||
}
|
||||
return scanner.WithMarkerAuditContext(r.Context(), audit)
|
||||
}
|
||||
|
||||
// HandleGetFileMarkers returns the current markers + provenance for a file.
|
||||
func (h *MarkersHandler) HandleGetFileMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(file))
|
||||
}
|
||||
|
||||
// HandleGetItemMarkers returns markers for the item's primary file.
|
||||
func (h *MarkersHandler) HandleGetItemMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadItemPrimaryFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(file))
|
||||
}
|
||||
|
||||
// HandleSetFileMarkers upserts the manual marker layer. Each segment key may be
|
||||
// an object {start, end} to set, or null to clear; absent keys are unchanged.
|
||||
func (h *MarkersHandler) HandleSetFileMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
h.setMarkersForFile(w, r, file)
|
||||
}
|
||||
|
||||
// HandleSetItemMarkers upserts manual markers on the item's primary file.
|
||||
func (h *MarkersHandler) HandleSetItemMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadItemPrimaryFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
h.setMarkersForFile(w, r, file)
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) setMarkersForFile(w http.ResponseWriter, r *http.Request, file *models.MediaFile) {
|
||||
if h.Writer == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
duration := float64(file.Duration)
|
||||
update := scanner.MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceManual,
|
||||
MarkersConfidence: &manualMarkerConfidence,
|
||||
MarkersAlgorithm: manualMarkerAlgorithm,
|
||||
}
|
||||
var clears []string
|
||||
var setSegs []string
|
||||
|
||||
for _, seg := range markerSegmentNames {
|
||||
val, present := raw[seg]
|
||||
if !present {
|
||||
continue
|
||||
}
|
||||
if isJSONNull(val) {
|
||||
clears = append(clears, seg)
|
||||
continue
|
||||
}
|
||||
var in segmentInput
|
||||
if err := json.Unmarshal(val, &in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid "+seg+" marker")
|
||||
return
|
||||
}
|
||||
start, end, err := normalizeManualSegment(seg, in, duration)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
applyManualSegment(&update, seg, start, end)
|
||||
setSegs = append(setSegs, seg)
|
||||
}
|
||||
|
||||
if _, err := h.Writer.UpsertAndClearMarkers(h.auditContext(r), file.ID, update, clears); err != nil {
|
||||
h.logger.Error("markers: save failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save markers")
|
||||
return
|
||||
}
|
||||
|
||||
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: reload after save failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Markers saved but failed to reload")
|
||||
return
|
||||
}
|
||||
h.maybeContribute(refreshed, setSegs)
|
||||
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
|
||||
}
|
||||
|
||||
// HandleClearFileSegment clears a single segment.
|
||||
func (h *MarkersHandler) HandleClearFileSegment(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Writer == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
segment := chi.URLParam(r, "segment")
|
||||
if !isMarkerSegment(segment) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Unknown marker segment")
|
||||
return
|
||||
}
|
||||
if _, err := h.Writer.ClearMarkers(h.auditContext(r), file.ID, []string{segment}); err != nil {
|
||||
h.logger.Error("markers: clear segment failed", "file_id", file.ID, "segment", segment, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to clear marker")
|
||||
return
|
||||
}
|
||||
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: reload after clear failed", "file_id", file.ID, "segment", segment, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Marker cleared but failed to reload")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
|
||||
}
|
||||
|
||||
// HandleContributeFile submits the file's eligible markers to enabled providers.
|
||||
func (h *MarkersHandler) HandleContributeFile(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Contributor == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Contribution is not configured")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Provider string `json:"provider"`
|
||||
Segments []string `json:"segments"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
var kinds []markers.MarkerKind
|
||||
for _, name := range body.Segments {
|
||||
kind, ok := markerKindForName(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Unknown segment "+name)
|
||||
return
|
||||
}
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
outcomes, err := h.Contributor.ContributeFile(r.Context(), file, markers.ContributeOptions{Provider: body.Provider, Segments: kinds})
|
||||
if err != nil {
|
||||
h.logger.Error("markers: contribute failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Contribution failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"outcomes": contributionOutcomeResponses(outcomes)})
|
||||
}
|
||||
|
||||
// HandleListFileContributions returns the contribution history for a file.
|
||||
func (h *MarkersHandler) HandleListFileContributions(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Contributions == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"contributions": []contributionRowResponse{}})
|
||||
return
|
||||
}
|
||||
rows, err := h.Contributions.ListByFile(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list contributions failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load contributions")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"contributions": contributionRowResponses(rows)})
|
||||
}
|
||||
|
||||
// HandleListFileMarkerHistory returns recent semantic marker edit audit rows for one file.
|
||||
func (h *MarkersHandler) HandleListFileMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
h.listMarkerHistory(w, r, []int{file.ID})
|
||||
}
|
||||
|
||||
// HandleListItemMarkerHistory returns recent marker edit audit rows for every file version on an item.
|
||||
func (h *MarkersHandler) HandleListItemMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker history is not configured")
|
||||
return
|
||||
}
|
||||
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if itemID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
|
||||
return
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: episode history file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return
|
||||
}
|
||||
if len(files) == 0 {
|
||||
files, err = h.Files.GetByContentID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: content history file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fileIDs := make([]int, 0, len(files))
|
||||
for _, file := range files {
|
||||
if file != nil {
|
||||
fileIDs = append(fileIDs, file.ID)
|
||||
}
|
||||
}
|
||||
if len(fileIDs) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return
|
||||
}
|
||||
h.listMarkerHistory(w, r, fileIDs)
|
||||
}
|
||||
|
||||
// HandleListMarkerHistory returns recent semantic marker edit audit rows across all files.
|
||||
func (h *MarkersHandler) HandleListMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
if h.AuditHistory == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
|
||||
return
|
||||
}
|
||||
limit, ok := markerHistoryLimit(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.AuditHistory.ListAllMarkerEditAudit(r.Context(), limit)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list all history failed", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) listMarkerHistory(w http.ResponseWriter, r *http.Request, fileIDs []int) {
|
||||
if h.AuditHistory == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
|
||||
return
|
||||
}
|
||||
limit, ok := markerHistoryLimit(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.AuditHistory.ListMarkerEditAudit(r.Context(), fileIDs, limit)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list history failed", "file_ids", fileIDs, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
|
||||
}
|
||||
|
||||
func markerHistoryLimit(w http.ResponseWriter, r *http.Request) (int, bool) {
|
||||
limit := 25
|
||||
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil || parsed <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid limit is required")
|
||||
return 0, false
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
return limit, true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) reloadAndNotify(ctx context.Context, fileID int) (*models.MediaFile, error) {
|
||||
refreshed, err := h.Files.GetByID(ctx, fileID)
|
||||
if err != nil || refreshed == nil {
|
||||
if err == nil {
|
||||
err = scanner.ErrFileNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if h.Notifier != nil {
|
||||
h.Notifier.MarkersUpdated(ctx, refreshed)
|
||||
}
|
||||
return refreshed, nil
|
||||
}
|
||||
|
||||
// maybeContribute fires a best-effort, detached contribution run for the
|
||||
// segments just set by a manual save. ContributeFile self-gates on each
|
||||
// provider's contribute_enabled flag, only submits scanner/manual-sourced
|
||||
// segments, and is idempotent — so this is a no-op when contribution is off or
|
||||
// the marker was already submitted. It runs in the background so the save
|
||||
// responds immediately and is never blocked by provider rate limits.
|
||||
func (h *MarkersHandler) maybeContribute(file *models.MediaFile, segments []string) {
|
||||
if h == nil || h.Contributor == nil || file == nil || len(segments) == 0 {
|
||||
return
|
||||
}
|
||||
kinds := make([]markers.MarkerKind, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
if kind, ok := markerKindForName(seg); ok {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
}
|
||||
if len(kinds) == 0 {
|
||||
return
|
||||
}
|
||||
base := h.BaseContext
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(base, markerContributeTimeout)
|
||||
defer cancel()
|
||||
if _, err := h.Contributor.ContributeFile(ctx, file, markers.ContributeOptions{Segments: kinds}); err != nil {
|
||||
h.logger.Warn("markers: background contribution failed", "file_id", file.ID, "error", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func isJSONNull(raw json.RawMessage) bool {
|
||||
return string(bytes.TrimSpace(raw)) == "null"
|
||||
}
|
||||
|
||||
func isMarkerSegment(seg string) bool {
|
||||
_, ok := markerKindForName(seg)
|
||||
return ok
|
||||
}
|
||||
|
||||
func markerKindForName(name string) (markers.MarkerKind, bool) {
|
||||
switch name {
|
||||
case "intro":
|
||||
return markers.MarkerKindIntro, true
|
||||
case "credits":
|
||||
return markers.MarkerKindCredits, true
|
||||
case "recap":
|
||||
return markers.MarkerKindRecap, true
|
||||
case "preview":
|
||||
return markers.MarkerKindPreview, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func markerNameForKind(kind markers.MarkerKind) string {
|
||||
switch kind {
|
||||
case markers.MarkerKindIntro:
|
||||
return "intro"
|
||||
case markers.MarkerKindCredits:
|
||||
return "credits"
|
||||
case markers.MarkerKindRecap:
|
||||
return "recap"
|
||||
case markers.MarkerKindPreview:
|
||||
return "preview"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func contributionOutcomeResponses(outcomes []markers.ContributionOutcome) []contributionOutcomeResponse {
|
||||
resp := make([]contributionOutcomeResponse, 0, len(outcomes))
|
||||
for _, o := range outcomes {
|
||||
item := contributionOutcomeResponse{
|
||||
Provider: o.Provider,
|
||||
Segment: markerNameForKind(o.Segment),
|
||||
Status: o.Status,
|
||||
SubmissionID: o.SubmissionID,
|
||||
Reason: o.Reason,
|
||||
}
|
||||
if o.RetryAfter > 0 {
|
||||
item.RetryAfterSeconds = int(o.RetryAfter.Seconds())
|
||||
}
|
||||
resp = append(resp, item)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func contributionRowResponses(rows []markers.ContributionRow) []contributionRowResponse {
|
||||
resp := make([]contributionRowResponse, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
item := contributionRowResponse{
|
||||
ID: row.ID,
|
||||
MediaFileID: row.MediaFileID,
|
||||
Provider: row.Provider,
|
||||
Segment: row.SegmentKind,
|
||||
Source: row.Source,
|
||||
SubmittedStartMS: row.SubmittedStartMs,
|
||||
SubmittedEndMS: row.SubmittedEndMs,
|
||||
VideoDurationMS: row.VideoDurationMs,
|
||||
ContentHash: row.ContentHash,
|
||||
SubmissionID: row.SubmissionID,
|
||||
Status: row.Status,
|
||||
HTTPStatus: row.HTTPStatus,
|
||||
Error: row.Error,
|
||||
}
|
||||
if !row.SubmittedAt.IsZero() {
|
||||
submittedAt := row.SubmittedAt
|
||||
item.SubmittedAt = &submittedAt
|
||||
}
|
||||
if !row.UpdatedAt.IsZero() {
|
||||
updatedAt := row.UpdatedAt
|
||||
item.UpdatedAt = &updatedAt
|
||||
}
|
||||
resp = append(resp, item)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func markerEditAuditResponses(rows []scanner.MarkerEditAuditRow) []markerEditAuditResponse {
|
||||
resp := make([]markerEditAuditResponse, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
resp = append(resp, markerEditAuditResponse{
|
||||
ID: row.ID,
|
||||
MediaFileID: row.MediaFileID,
|
||||
ItemID: row.ItemID,
|
||||
ItemType: row.ItemType,
|
||||
MediaTitle: row.MediaTitle,
|
||||
FilePath: row.FilePath,
|
||||
Segment: row.SegmentKind,
|
||||
Action: row.Action,
|
||||
Before: markerAuditSegmentResponse(row.Before),
|
||||
After: markerAuditSegmentResponse(row.After),
|
||||
UserID: row.UserID,
|
||||
Username: row.Username,
|
||||
ImpersonatorUserID: row.ImpersonatorUserID,
|
||||
ImpersonatorUsername: row.ImpersonatorUsername,
|
||||
APIKeyID: row.APIKeyID,
|
||||
RequestID: row.RequestID,
|
||||
ClientIP: row.ClientIP,
|
||||
UserAgent: row.UserAgent,
|
||||
CreatedAt: row.CreatedAt,
|
||||
})
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func markerAuditSegmentResponse(segment *scanner.MarkerAuditSegment) *segmentMarker {
|
||||
if segment == nil {
|
||||
return nil
|
||||
}
|
||||
return &segmentMarker{
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Source: segment.Source,
|
||||
Provider: segment.Provider,
|
||||
Confidence: segment.Confidence,
|
||||
Algorithm: segment.Algorithm,
|
||||
DetectedAt: segment.DetectedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeManualSegment applies the start/end defaults and validation, mirroring
|
||||
// the contribution rules: intro/recap may omit start (=0); credits/preview may
|
||||
// omit end (=duration); end must exceed start and stay within the file.
|
||||
func normalizeManualSegment(seg string, in segmentInput, duration float64) (start, end float64, err error) {
|
||||
switch seg {
|
||||
case "intro", "recap":
|
||||
if in.Start != nil {
|
||||
start = *in.Start
|
||||
}
|
||||
if in.End == nil {
|
||||
return 0, 0, errSegment(seg, "end is required")
|
||||
}
|
||||
end = *in.End
|
||||
default: // credits, preview
|
||||
if in.Start == nil {
|
||||
return 0, 0, errSegment(seg, "start is required")
|
||||
}
|
||||
start = *in.Start
|
||||
if in.End != nil {
|
||||
end = *in.End
|
||||
} else if duration > 0 {
|
||||
end = duration
|
||||
} else {
|
||||
return 0, 0, errSegment(seg, "end is required when duration is unknown")
|
||||
}
|
||||
}
|
||||
if start < 0 || end <= start {
|
||||
return 0, 0, errSegment(seg, "end must be greater than start")
|
||||
}
|
||||
if duration > 0 && end > duration+1 {
|
||||
return 0, 0, errSegment(seg, "end exceeds the file duration")
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
func applyManualSegment(update *scanner.MarkerUpdate, seg string, start, end float64) {
|
||||
s, e := start, end
|
||||
switch seg {
|
||||
case "intro":
|
||||
update.IntroStart, update.IntroEnd = &s, &e
|
||||
case "credits":
|
||||
update.CreditsStart, update.CreditsEnd = &s, &e
|
||||
case "recap":
|
||||
update.RecapStart, update.RecapEnd = &s, &e
|
||||
case "preview":
|
||||
update.PreviewStart, update.PreviewEnd = &s, &e
|
||||
}
|
||||
}
|
||||
|
||||
func errSegment(seg, msg string) error {
|
||||
return &markerValidationError{seg: seg, msg: msg}
|
||||
}
|
||||
|
||||
type markerValidationError struct {
|
||||
seg string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *markerValidationError) Error() string { return e.seg + " marker: " + e.msg }
|
||||
@@ -1,524 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
)
|
||||
|
||||
type fakeMarkerFiles struct {
|
||||
file *models.MediaFile
|
||||
byID map[int]*models.MediaFile
|
||||
contentFiles []*models.MediaFile
|
||||
episodeFiles []*models.MediaFile
|
||||
}
|
||||
|
||||
func (f fakeMarkerFiles) GetByID(_ context.Context, id int) (*models.MediaFile, error) {
|
||||
if f.byID != nil {
|
||||
return f.byID[id], nil
|
||||
}
|
||||
return f.file, nil
|
||||
}
|
||||
func (f fakeMarkerFiles) GetByContentID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return f.contentFiles, nil
|
||||
}
|
||||
func (f fakeMarkerFiles) GetByEpisodeID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return f.episodeFiles, nil
|
||||
}
|
||||
|
||||
type fakeMarkerWriter struct {
|
||||
upserts []scanner.MarkerUpdate
|
||||
clears [][]string
|
||||
audits []scanner.MarkerAuditContext
|
||||
}
|
||||
|
||||
func (f *fakeMarkerWriter) captureAudit(ctx context.Context) {
|
||||
if audit, ok := scanner.MarkerAuditContextFromContext(ctx); ok {
|
||||
f.audits = append(f.audits, audit)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeMarkerWriter) UpsertMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
f.upserts = append(f.upserts, u)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeMarkerWriter) ClearMarkers(ctx context.Context, _ int, segs []string) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
f.clears = append(f.clears, segs)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeMarkerWriter) UpsertAndClearMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate, segs []string) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
if u.HasAnySegment() {
|
||||
f.upserts = append(f.upserts, u)
|
||||
}
|
||||
if len(segs) > 0 {
|
||||
f.clears = append(f.clears, segs)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func markerPutRequest(body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/markers/files/5", strings.NewReader(body))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
}
|
||||
|
||||
func markerItemPutRequest(body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/markers/items/episode-1", strings.NewReader(body))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "episode-1")
|
||||
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
}
|
||||
|
||||
func withMarkerAdminClaims(req *http.Request) *http.Request {
|
||||
return req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{
|
||||
UserID: 1,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
SessionID: "session-1",
|
||||
}))
|
||||
}
|
||||
|
||||
func newMarkersHandler(writer ManualMarkerWriter) *MarkersHandler {
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
return NewMarkersHandler(files, writer, nil, nil, nil, nil)
|
||||
}
|
||||
|
||||
func TestSetFileMarkersWritesManual(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
|
||||
}
|
||||
u := writer.upserts[0]
|
||||
if u.MarkersSource != models.MarkerSourceManual {
|
||||
t.Errorf("source = %q, want manual", u.MarkersSource)
|
||||
}
|
||||
if u.IntroStart == nil || *u.IntroStart != 0 || u.IntroEnd == nil || *u.IntroEnd != 60 {
|
||||
t.Errorf("intro = %v..%v, want 0..60", u.IntroStart, u.IntroEnd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetItemMarkersWritesPrimaryEpisodeFile(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
file := &models.MediaFile{ID: 8, Duration: 1800}
|
||||
files := fakeMarkerFiles{
|
||||
byID: map[int]*models.MediaFile{8: file},
|
||||
episodeFiles: []*models.MediaFile{file},
|
||||
}
|
||||
h := NewMarkersHandler(files, writer, nil, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetItemMarkers(rec, markerItemPutRequest(`{"recap":{"start":0,"end":45}}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].RecapEnd == nil || *writer.upserts[0].RecapEnd != 45 {
|
||||
t.Errorf("recap end = %v, want 45", writer.upserts[0].RecapEnd)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeMarkerItemAccess map[string]error
|
||||
|
||||
func (f fakeMarkerItemAccess) EnsureAccessible(_ context.Context, contentID string, _ catalog.AccessFilter) error {
|
||||
return f[contentID]
|
||||
}
|
||||
|
||||
type fakeMarkerEpisodeLookup map[string]*models.Episode
|
||||
|
||||
func (f fakeMarkerEpisodeLookup) GetByID(_ context.Context, contentID string) (*models.Episode, error) {
|
||||
return f[contentID], nil
|
||||
}
|
||||
|
||||
type fakeMarkerUsers map[int]*models.User
|
||||
|
||||
func (f fakeMarkerUsers) GetByID(_ context.Context, id int) (*models.User, error) {
|
||||
return f[id], nil
|
||||
}
|
||||
|
||||
func TestGetItemMarkersUsesFirstAuthorizedFile(t *testing.T) {
|
||||
denied := &models.MediaFile{ID: 8, EpisodeID: "episode-denied", Duration: 1800}
|
||||
allowed := &models.MediaFile{ID: 9, EpisodeID: "episode-allowed", Duration: 1800}
|
||||
files := fakeMarkerFiles{
|
||||
byID: map[int]*models.MediaFile{
|
||||
8: denied,
|
||||
9: allowed,
|
||||
},
|
||||
episodeFiles: []*models.MediaFile{denied, allowed},
|
||||
}
|
||||
h := NewMarkersHandler(files, nil, nil, nil, nil, nil)
|
||||
h.Authorizer = &MediaFileAuthorizer{
|
||||
FileResolver: files,
|
||||
ItemAccess: fakeMarkerItemAccess{
|
||||
"series-denied": catalog.ErrItemNotFound,
|
||||
},
|
||||
EpisodeLookup: fakeMarkerEpisodeLookup{
|
||||
"episode-denied": {ContentID: "episode-denied", SeriesID: "series-denied"},
|
||||
"episode-allowed": {ContentID: "episode-allowed", SeriesID: "series-allowed"},
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/markers/items/episode-1", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "episode-1")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetItemMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body fileMarkersResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if body.FileID != allowed.ID {
|
||||
t.Fatalf("file_id = %d, want first authorized file %d", body.FileID, allowed.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetFileMarkersDoesNotRequireMarkerEditPermission(t *testing.T) {
|
||||
h := newMarkersHandler(nil)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/markers/files/5", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersRejectsUserWithoutMarkerEditPermission(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
|
||||
}
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("permission failure wrote markers: %+v", writer.upserts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersAllowsUserWithMarkerEditPermission(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: []string{"marker_edit"}},
|
||||
}
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected marker write, got %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersPassesAuditContextToWriter(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
apiKeyID := int64(99)
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req.Header.Set("User-Agent", "marker-test-agent")
|
||||
ctx := clientip.SetContext(req.Context(), "203.0.113.10")
|
||||
ctx = apimw.SetClaims(ctx, &auth.Claims{
|
||||
UserID: 7,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAPIKey,
|
||||
APIKeyID: apiKeyID,
|
||||
})
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.audits) != 1 {
|
||||
t.Fatalf("audit contexts = %d, want 1", len(writer.audits))
|
||||
}
|
||||
audit := writer.audits[0]
|
||||
if audit.UserID == nil || *audit.UserID != 7 {
|
||||
t.Fatalf("audit user id = %v, want 7", audit.UserID)
|
||||
}
|
||||
if audit.APIKeyID == nil || *audit.APIKeyID != apiKeyID {
|
||||
t.Fatalf("audit api key id = %v, want %d", audit.APIKeyID, apiKeyID)
|
||||
}
|
||||
if audit.ClientIP != "203.0.113.10" || audit.UserAgent != "marker-test-agent" {
|
||||
t.Fatalf("audit request metadata = ip %q ua %q", audit.ClientIP, audit.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersClearsOnNull(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Errorf("expected no upsert for a null clear, got %d", len(writer.upserts))
|
||||
}
|
||||
if len(writer.clears) != 1 || len(writer.clears[0]) != 1 || writer.clears[0][0] != "credits" {
|
||||
t.Errorf("clears = %v, want [[credits]]", writer.clears)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersRejectsInvalidRange(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":60,"end":10}}`))
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Errorf("invalid range must not write, got %d upserts", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearFileSegmentRejectsUnknown(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/markers/files/5/bogus", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
rctx.URLParams.Add("segment", "bogus")
|
||||
req = withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleClearFileSegment(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeMarkerContributor struct{ outcomes []markers.ContributionOutcome }
|
||||
|
||||
func (f fakeMarkerContributor) ContributeFile(context.Context, *models.MediaFile, markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
return f.outcomes, nil
|
||||
}
|
||||
|
||||
type captureMarkerContributor struct {
|
||||
opts markers.ContributeOptions
|
||||
}
|
||||
|
||||
func (c *captureMarkerContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
c.opts = opts
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// signalContributor reports every ContributeFile call on a channel so a test
|
||||
// can wait for the detached background contribution kicked off by a save.
|
||||
type signalContributor struct {
|
||||
called chan markers.ContributeOptions
|
||||
}
|
||||
|
||||
func (s signalContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
s.called <- opts
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestSetFileMarkersTriggersBackgroundContribution(t *testing.T) {
|
||||
called := make(chan markers.ContributeOptions, 1)
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case opts := <-called:
|
||||
if len(opts.Segments) != 1 || opts.Segments[0] != markers.MarkerKindIntro {
|
||||
t.Fatalf("contribute segments = %v, want [intro]", opts.Segments)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("expected a background contribution for the saved intro segment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersClearOnlyDoesNotContribute(t *testing.T) {
|
||||
called := make(chan markers.ContributeOptions, 1)
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-called:
|
||||
t.Fatal("a clear-only save must not trigger contribution")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
type fakeContributionLister struct{ rows []markers.ContributionRow }
|
||||
|
||||
func (f fakeContributionLister) ListByFile(context.Context, int) ([]markers.ContributionRow, error) {
|
||||
return f.rows, nil
|
||||
}
|
||||
|
||||
func TestContributeFileUsesSnakeCaseResponse(t *testing.T) {
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
fakeMarkerContributor{outcomes: []markers.ContributionOutcome{{
|
||||
Provider: "introdb", Segment: markers.MarkerKindCredits, Status: markers.OutcomeStatusRateLimited,
|
||||
Reason: "usage limited", RetryAfter: 30 * time.Second,
|
||||
}}},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleContributeFile(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Outcomes []map[string]any `json:"outcomes"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(body.Outcomes) != 1 || body.Outcomes[0]["segment"] != "credits" {
|
||||
t.Fatalf("outcomes = %+v, want credits segment name", body.Outcomes)
|
||||
}
|
||||
if _, ok := body.Outcomes[0]["Segment"]; ok {
|
||||
t.Fatalf("response leaked internal Segment field: %s", rec.Body.String())
|
||||
}
|
||||
if body.Outcomes[0]["retry_after_seconds"] != float64(30) {
|
||||
t.Fatalf("retry_after_seconds = %v, want 30", body.Outcomes[0]["retry_after_seconds"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeFileDecodesUnknownLengthBody(t *testing.T) {
|
||||
contributor := &captureMarkerContributor{}
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
contributor,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{"provider":"introdb","segments":["credits"]}`))
|
||||
req.ContentLength = -1
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleContributeFile(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if contributor.opts.Provider != "introdb" {
|
||||
t.Fatalf("provider = %q, want introdb", contributor.opts.Provider)
|
||||
}
|
||||
if len(contributor.opts.Segments) != 1 || contributor.opts.Segments[0] != markers.MarkerKindCredits {
|
||||
t.Fatalf("segments = %+v, want [credits]", contributor.opts.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFileContributionsUsesSnakeCaseResponse(t *testing.T) {
|
||||
now := time.Unix(100, 0).UTC()
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
nil,
|
||||
fakeContributionLister{rows: []markers.ContributionRow{{
|
||||
ID: "row1", MediaFileID: 5, Provider: "introdb", SegmentKind: "intro",
|
||||
Source: "manual", ContentHash: "hash", Status: "pending", SubmittedAt: now, UpdatedAt: now,
|
||||
}}},
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/files/5/contributions", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListFileContributions(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"media_file_id":5`) || strings.Contains(body, "MediaFileID") {
|
||||
t.Fatalf("unexpected contribution response shape: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -2257,7 +2257,6 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2271,7 +2270,6 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2286,7 +2284,6 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2298,7 +2295,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
// active encode range), either restart at the exact manifest-derived
|
||||
// timeline position or return 404 for copy-mode segments outside the
|
||||
// current manifest window.
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) && decision.RestartOnTimeout {
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
|
||||
seekSeconds, ok, seekErr := transcodeSession.RestartSeekTarget(segNum)
|
||||
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
|
||||
slog.Error("resolve transcode seek target", "error", seekErr, "segment", segmentName, "session", sessionID, "playback_session_id", sessionID)
|
||||
@@ -2313,7 +2310,6 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"seek_seconds", seekSeconds,
|
||||
"session", sessionID,
|
||||
|
||||
@@ -272,7 +272,7 @@ func (h *PlaybackHandler) fetchOnlineMarkersForPlayback(ctx context.Context, fil
|
||||
EpisodeNumber: ids.EpisodeNumber,
|
||||
Duration: time.Duration(file.Duration) * time.Second,
|
||||
}
|
||||
result, ok, err := h.MarkerRegistry.FetchMerged(ctx, req)
|
||||
result, ok, err := h.MarkerRegistry.FetchFirstHit(ctx, req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -373,31 +373,17 @@ func hasLocalDetectionMarkers(file *models.MediaFile) bool {
|
||||
// each other.
|
||||
func markerUpdateFromPayload(p markers.MarkerUpdatePayload) scanner.MarkerUpdate {
|
||||
return scanner.MarkerUpdate{
|
||||
IntroStart: p.Intro.Start,
|
||||
IntroEnd: p.Intro.End,
|
||||
CreditsStart: p.Credits.Start,
|
||||
CreditsEnd: p.Credits.End,
|
||||
RecapStart: p.Recap.Start,
|
||||
RecapEnd: p.Recap.End,
|
||||
PreviewStart: p.Preview.Start,
|
||||
PreviewEnd: p.Preview.End,
|
||||
MarkersSource: p.SummarySource(),
|
||||
MarkersConfidence: p.SummaryConfidence(),
|
||||
IntroProvenance: segmentProvenance(p.Intro),
|
||||
CreditsProvenance: segmentProvenance(p.Credits),
|
||||
RecapProvenance: segmentProvenance(p.Recap),
|
||||
PreviewProvenance: segmentProvenance(p.Preview),
|
||||
}
|
||||
}
|
||||
|
||||
func segmentProvenance(s markers.SegmentPayload) *scanner.SegmentProvenance {
|
||||
if !s.Present() {
|
||||
return nil
|
||||
}
|
||||
return &scanner.SegmentProvenance{
|
||||
Source: s.Source,
|
||||
Provider: s.Provider,
|
||||
Confidence: s.Confidence,
|
||||
Algorithm: s.Algorithm,
|
||||
IntroStart: p.IntroStart,
|
||||
IntroEnd: p.IntroEnd,
|
||||
CreditsStart: p.CreditsStart,
|
||||
CreditsEnd: p.CreditsEnd,
|
||||
RecapStart: p.RecapStart,
|
||||
RecapEnd: p.RecapEnd,
|
||||
PreviewStart: p.PreviewStart,
|
||||
PreviewEnd: p.PreviewEnd,
|
||||
MarkersSource: p.Source,
|
||||
MarkersProvider: p.Provider,
|
||||
MarkersConfidence: p.Confidence,
|
||||
MarkersAlgorithm: p.Algorithm,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ type SectionHandler struct {
|
||||
StoreProvider userstore.UserStoreProvider
|
||||
UserRepo *auth.UserRepository
|
||||
DetailSvc *catalog.DetailService
|
||||
Settings catalog.SettingsStore
|
||||
Settings *catalog.ServerSettingsRepo
|
||||
CollectionRepo *catalog.LibraryCollectionRepository
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ const SectionsAllowProfileCustomSettingKey = "sections.allow_profile_custom_sect
|
||||
|
||||
// SectionSettingsHandler exposes GET/PUT for the sections-related server setting.
|
||||
type SectionSettingsHandler struct {
|
||||
Settings catalog.SettingsStore
|
||||
Settings *catalog.ServerSettingsRepo
|
||||
}
|
||||
|
||||
type sectionsSettingResponse struct {
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
var ErrServerRestartAlreadyRequested = errors.New("server restart already requested")
|
||||
|
||||
type ServerControlHandler struct {
|
||||
requestRestart func(ctx context.Context) error
|
||||
commands *playback.CommandDispatcher
|
||||
}
|
||||
|
||||
type serverRestartRequest struct {
|
||||
Reason string `json:"reason"`
|
||||
Title string `json:"title"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type serverRestartResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
NotifiedSessions int `json:"notified_sessions"`
|
||||
}
|
||||
|
||||
func NewServerControlHandler(
|
||||
requestRestart func(ctx context.Context) error,
|
||||
commands *playback.CommandDispatcher,
|
||||
) *ServerControlHandler {
|
||||
return &ServerControlHandler{
|
||||
requestRestart: requestRestart,
|
||||
commands: commands,
|
||||
}
|
||||
}
|
||||
|
||||
// HandleRestart handles POST /admin/server/restart.
|
||||
func (h *ServerControlHandler) HandleRestart(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.requestRestart == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Server restart is unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
var req serverRestartRequest
|
||||
if err := decodeOptionalJSONBody(r, &req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
notifiedSessions := h.notifyPlaybackSessions(req)
|
||||
|
||||
if err := h.requestRestart(context.Background()); err != nil {
|
||||
if errors.Is(err, ErrServerRestartAlreadyRequested) {
|
||||
writeJSON(w, http.StatusAccepted, serverRestartResponse{
|
||||
Status: "already_requested",
|
||||
Message: "Server restart is already in progress.",
|
||||
NotifiedSessions: notifiedSessions,
|
||||
})
|
||||
return
|
||||
}
|
||||
slog.Error("server restart request failed", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to request server restart")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, serverRestartResponse{
|
||||
Status: "restart_requested",
|
||||
Message: "Server restart requested. The process will shut down gracefully.",
|
||||
NotifiedSessions: notifiedSessions,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ServerControlHandler) notifyPlaybackSessions(req serverRestartRequest) int {
|
||||
if h == nil || h.commands == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
title := req.Title
|
||||
if title == "" {
|
||||
title = "Server restarting"
|
||||
}
|
||||
message := req.Message
|
||||
if message == "" {
|
||||
message = "The server is restarting now. Playback may reconnect shortly."
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(map[string]string{
|
||||
"title": title,
|
||||
"message": message,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Warn("server restart: failed to build realtime payload", "error", err)
|
||||
return 0
|
||||
}
|
||||
|
||||
results := h.commands.DispatchToAll(func(session *playback.Session) (playback.CommandEnvelope, time.Duration, func(), error) {
|
||||
if session == nil {
|
||||
return playback.CommandEnvelope{}, 0, nil, playback.ErrCommandDispatchUnavailable
|
||||
}
|
||||
command, err := playback.NewCommandEnvelope(
|
||||
session.ID,
|
||||
uuid.NewString(),
|
||||
playback.CommandServerRestarting,
|
||||
payload,
|
||||
)
|
||||
if err != nil {
|
||||
return playback.CommandEnvelope{}, 0, nil, err
|
||||
}
|
||||
command.Reason = req.Reason
|
||||
if command.Reason == "" {
|
||||
command.Reason = "server_restart_requested"
|
||||
}
|
||||
command.IssuedBy = &playback.CommandIssuedBy{Kind: "admin"}
|
||||
return command, 0, nil, nil
|
||||
})
|
||||
|
||||
delivered := 0
|
||||
for _, result := range results {
|
||||
if result.Delivered {
|
||||
delivered++
|
||||
continue
|
||||
}
|
||||
if result.DispatchErr != nil && !errors.Is(result.DispatchErr, playback.ErrRealtimeConnectionNotFound) {
|
||||
slog.Warn("server restart: failed to notify playback session",
|
||||
"session_id", result.SessionID,
|
||||
"error", result.DispatchErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
return delivered
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
type serverControlTestConn struct {
|
||||
messages []any
|
||||
}
|
||||
|
||||
func (c *serverControlTestConn) WriteJSON(v any) error {
|
||||
c.messages = append(c.messages, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestServerControlRestartRequestsShutdown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
called := 0
|
||||
handler := NewServerControlHandler(func(context.Context) error {
|
||||
called++
|
||||
return nil
|
||||
}, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/server/restart", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleRestart(rec, req)
|
||||
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if called != 1 {
|
||||
t.Fatalf("restart calls = %d, want 1", called)
|
||||
}
|
||||
|
||||
var resp serverRestartResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.Status != "restart_requested" {
|
||||
t.Fatalf("status = %q, want restart_requested", resp.Status)
|
||||
}
|
||||
if resp.NotifiedSessions != 0 {
|
||||
t.Fatalf("notified_sessions = %d, want 0", resp.NotifiedSessions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerControlRestartUnavailable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
handler := NewServerControlHandler(nil, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/server/restart", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleRestart(rec, req)
|
||||
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerControlRestartAlreadyRequested(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
handler := NewServerControlHandler(func(context.Context) error {
|
||||
return ErrServerRestartAlreadyRequested
|
||||
}, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/server/restart", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleRestart(rec, req)
|
||||
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp serverRestartResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.Status != "already_requested" {
|
||||
t.Fatalf("status = %q, want already_requested", resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerControlRestartNotifiesPlaybackSessions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sessionMgr := playback.NewSessionManager(0, 0)
|
||||
session, err := sessionMgr.StartSession(1, "profile-1", 100, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
|
||||
realtimeHub := playback.NewRealtimeHub()
|
||||
conn := &serverControlTestConn{}
|
||||
registration := realtimeHub.Register(session.ID, conn)
|
||||
if registration == nil {
|
||||
t.Fatal("expected realtime registration")
|
||||
}
|
||||
defer realtimeHub.Unregister(registration)
|
||||
|
||||
dispatcher := playback.NewCommandDispatcher(sessionMgr, realtimeHub, nil)
|
||||
handler := NewServerControlHandler(func(context.Context) error {
|
||||
return nil
|
||||
}, dispatcher)
|
||||
|
||||
req := httptest.NewRequest(
|
||||
http.MethodPost,
|
||||
"/admin/server/restart",
|
||||
strings.NewReader(`{"reason":"maintenance","message":"Restarting for maintenance"}`),
|
||||
)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleRestart(rec, req)
|
||||
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp serverRestartResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.NotifiedSessions != 1 {
|
||||
t.Fatalf("notified_sessions = %d, want 1", resp.NotifiedSessions)
|
||||
}
|
||||
if len(conn.messages) != 1 {
|
||||
t.Fatalf("messages = %d, want 1", len(conn.messages))
|
||||
}
|
||||
|
||||
command, ok := conn.messages[0].(playback.CommandEnvelope)
|
||||
if !ok {
|
||||
t.Fatalf("message type = %T, want playback.CommandEnvelope", conn.messages[0])
|
||||
}
|
||||
if command.Name != playback.CommandServerRestarting {
|
||||
t.Fatalf("command name = %q, want %q", command.Name, playback.CommandServerRestarting)
|
||||
}
|
||||
if command.Reason != "maintenance" {
|
||||
t.Fatalf("reason = %q, want maintenance", command.Reason)
|
||||
}
|
||||
|
||||
var payload map[string]string
|
||||
if err := json.Unmarshal(command.Payload, &payload); err != nil {
|
||||
t.Fatalf("decode payload: %v", err)
|
||||
}
|
||||
if payload["message"] != "Restarting for maintenance" {
|
||||
t.Fatalf("payload message = %q, want custom message", payload["message"])
|
||||
}
|
||||
}
|
||||
@@ -17,16 +17,14 @@ import (
|
||||
type SystemHandler struct {
|
||||
transcodePool *nodepool.TranscodePool
|
||||
jwtSecret string
|
||||
ffmpegPath string
|
||||
buildInfo buildinfo.Info
|
||||
}
|
||||
|
||||
// NewSystemHandler creates a SystemHandler.
|
||||
func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret string, ffmpegPath string) *SystemHandler {
|
||||
func NewSystemHandler(transcodePool *nodepool.TranscodePool, jwtSecret string) *SystemHandler {
|
||||
return &SystemHandler{
|
||||
transcodePool: transcodePool,
|
||||
jwtSecret: jwtSecret,
|
||||
ffmpegPath: ffmpegPath,
|
||||
buildInfo: buildinfo.Current(),
|
||||
}
|
||||
}
|
||||
@@ -47,7 +45,7 @@ func (h *SystemHandler) HandleHWAccel(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
info := playback.DetectHWAccelWithFFmpeg(h.ffmpegPath)
|
||||
info := playback.DetectHWAccel()
|
||||
writeJSON(w, http.StatusOK, info)
|
||||
}
|
||||
|
||||
|
||||
@@ -32,11 +32,11 @@ type tasteSeedSubmitResponse struct {
|
||||
|
||||
// HandleTasteSeedItems handles GET /recommendations/taste-seed/items.
|
||||
//
|
||||
// Returns a paginated, hydrated list of posters used for the new-user
|
||||
// taste-seeding picker. Blends server-watched popularity with rating reliability
|
||||
// and recency so fresh servers (no watch history yet) still surface recognizable
|
||||
// content. The user_state field carries the existing is_favorite flag, so the UI
|
||||
// can pre-select items the profile already favorited.
|
||||
// Returns a paginated, hydrated list of "popular on this server" posters used
|
||||
// for the new-user taste-seeding picker. Blends server-watched popularity with
|
||||
// TMDB rating so fresh servers (no watch history yet) still surface meaningful
|
||||
// content. The user_state field carries the existing is_favorite flag, so the
|
||||
// UI can pre-select items the profile already favorited.
|
||||
func (h *RecommendationsHandler) HandleTasteSeedItems(w http.ResponseWriter, r *http.Request) {
|
||||
if h.recsRepo == nil || h.Fetcher == nil {
|
||||
writeJSON(w, http.StatusOK, tasteSeedItemsResponse{Items: []sectionItemResponse{}})
|
||||
|
||||
+27
-115
@@ -53,7 +53,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/s3client"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/Silo-Server/silo-server/internal/scanqueue"
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
subtitleai "github.com/Silo-Server/silo-server/internal/subtitles/ai"
|
||||
@@ -77,7 +76,6 @@ type Dependencies struct {
|
||||
BootstrapSensitiveValues map[string]string
|
||||
AppContext context.Context
|
||||
DB *pgxpool.Pool
|
||||
SecretCipher *secret.Cipher // at-rest credential cipher (required when DB is set)
|
||||
FrontendFS fs.FS
|
||||
S3Public *s3client.Client // public assets bucket client (may be nil)
|
||||
S3Private *s3client.Client // private internal bucket client (may be nil)
|
||||
@@ -125,9 +123,6 @@ type Dependencies struct {
|
||||
IntroAnalyzer *intromarkers.Analyzer
|
||||
MarkerRegistry *markers.Registry
|
||||
MarkerResolver markers.ExternalIDResolver
|
||||
MarkerProviderConfig *markers.ProviderConfigStore
|
||||
MarkerContributionStore *markers.ContributionStore
|
||||
MarkerContributionService *markers.ContributionService
|
||||
WatchProviderService handlers.WatchProviderService
|
||||
PluginService *plugins.Service
|
||||
PluginHTTPProxy *plugins.HTTPProxy
|
||||
@@ -145,7 +140,6 @@ type Dependencies struct {
|
||||
PlaybackRealtimeHub *playback.RealtimeHub
|
||||
OnUserSessionsRevoked func(ctx context.Context, userID int)
|
||||
OnServerSettingUpdated func(ctx context.Context, key, value string)
|
||||
RequestServerRestart func(ctx context.Context) error
|
||||
|
||||
// UserCollectionSync handles per-profile imported collections (TMDB /
|
||||
// Trakt / MDBList) — the user-facing analogue of CollectionService.
|
||||
@@ -167,6 +161,9 @@ type Dependencies struct {
|
||||
// clients hitting /login, /api/*, /abs/api/*, and /abs/socket.io/* all
|
||||
// resolve correctly. May be nil; no ABS routes are registered in that case.
|
||||
ABSHandler absHandler
|
||||
// AudiobooksEnabled gates native audiobook admin/UI routes behind the same
|
||||
// server setting as the ABS compatibility listener.
|
||||
AudiobooksEnabled bool
|
||||
}
|
||||
|
||||
// absHandler is the narrow interface the router needs from the ABS handler.
|
||||
@@ -225,11 +222,9 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
healthHandler := handlers.NewHealthHandler(healthServerName, healthServerID)
|
||||
|
||||
// Build server settings repo if DB is available (needed by auth and admin).
|
||||
// Wrap it in the encrypting decorator so sensitive keys rest as ciphertext
|
||||
// and every consumer transparently reads plaintext.
|
||||
var settingsRepo catalog.SettingsStore
|
||||
var settingsRepo *catalog.ServerSettingsRepo
|
||||
if deps.DB != nil {
|
||||
settingsRepo = catalog.NewEncryptedSettingsRepo(catalog.NewServerSettingsRepo(deps.DB), deps.SecretCipher)
|
||||
settingsRepo = catalog.NewServerSettingsRepo(deps.DB)
|
||||
}
|
||||
|
||||
// Build auth handler and auth middleware if DB and config are available.
|
||||
@@ -435,25 +430,27 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
if deps.Config != nil {
|
||||
tmdbAPIKey = deps.Config.TMDBAPIKey
|
||||
}
|
||||
requestsRepo := mediarequests.NewRepository(deps.DB, deps.SecretCipher)
|
||||
requestsRepo := mediarequests.NewRepository(deps.DB)
|
||||
requestSvc := mediarequests.NewService(
|
||||
requestsRepo,
|
||||
tmdb.NewClient(tmdbAPIKey, 40),
|
||||
mediarequests.NewCatalogPresence(itemRepo, providerIDRepo),
|
||||
)
|
||||
requestSvc.SetSecretResolver(settingsRepo)
|
||||
requestSvc.SetFulfillmentAdapters(radarr.NewClient(nil), sonarr.NewClient(nil))
|
||||
if viewerResolver != nil {
|
||||
requestSvc.SetEntitlementResolver(mediarequests.NewAccessEntitlements(viewerResolver))
|
||||
}
|
||||
requestHandler = handlers.NewRequestsHandler(requestSvc)
|
||||
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB, deps.SecretCipher)
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB)
|
||||
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil && deps.PluginService != nil {
|
||||
autoscanSvc := BuildAutoscanService(
|
||||
autoscanRepo,
|
||||
deps.PluginService,
|
||||
plugins.NewInstallationStore(deps.DB),
|
||||
requestsRepo,
|
||||
settingsRepo,
|
||||
deps.FolderRepo,
|
||||
deps.LibraryScanQueue,
|
||||
deps.RedisClient,
|
||||
@@ -568,7 +565,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// Create subtitleRepo early — only needs DB, shared with playback handler and subtitle search handler.
|
||||
var subtitleRepo *subtitles.PgRepository
|
||||
if deps.DB != nil {
|
||||
subtitleRepo = subtitles.NewPgRepository(deps.DB, deps.SecretCipher)
|
||||
subtitleRepo = subtitles.NewPgRepository(deps.DB)
|
||||
}
|
||||
|
||||
// Notifier that pushes "subtitle ready" events to active sessions when an AI
|
||||
@@ -579,10 +576,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// Build playback handler if session manager is available.
|
||||
var playbackHandler *handlers.PlaybackHandler
|
||||
var adminPlaybackControlHandler *handlers.AdminPlaybackControlHandler
|
||||
var playbackCommandDispatcher *playback.CommandDispatcher
|
||||
var streamHandler *handlers.StreamHandler
|
||||
var watchTogetherHandler *handlers.WatchTogetherHandler
|
||||
var audioHandler *handlers.AudioHandler
|
||||
if deps.SessionMgr != nil {
|
||||
var playbackAdminStore handlers.PlaybackAdminStore
|
||||
if deps.DB != nil {
|
||||
@@ -664,7 +659,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
playbackHandler.RealtimeHub = realtimeHub
|
||||
playbackHandler.CommandTracker = commandTracker
|
||||
playbackHandler.CommandDispatcher = playback.NewCommandDispatcher(deps.SessionMgr, realtimeHub, commandTracker)
|
||||
playbackCommandDispatcher = playbackHandler.CommandDispatcher
|
||||
playbackHandler.IntroAnalyzer = deps.IntroAnalyzer
|
||||
playbackHandler.IntroRepository = deps.IntroRepository
|
||||
playbackHandler.MarkerRegistry = deps.MarkerRegistry
|
||||
@@ -692,14 +686,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
)
|
||||
}
|
||||
}
|
||||
if detailSvc != nil && deps.FileRepo != nil && deps.UserStoreProvider != nil {
|
||||
audioHandler = handlers.NewAudioHandler(
|
||||
detailSvc,
|
||||
deps.FileRepo,
|
||||
deps.UserStoreProvider,
|
||||
handlers.NewPGAudioBookmarkStore(deps.DB),
|
||||
)
|
||||
}
|
||||
|
||||
// Wire subtitle repo and S3 client onto streamHandler for S3-stored subtitle serving.
|
||||
if streamHandler != nil && subtitleRepo != nil && deps.S3Public != nil {
|
||||
@@ -711,8 +697,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
streamHandler.FFmpegPath = deps.Config.Playback.FFmpegPath
|
||||
}
|
||||
|
||||
serverControlHandler := handlers.NewServerControlHandler(deps.RequestServerRestart, playbackCommandDispatcher)
|
||||
|
||||
// Build admin handler if we have a user repo.
|
||||
var adminHandler *handlers.AdminHandler
|
||||
var catalogSeedHandler *handlers.CatalogSeedHandler
|
||||
@@ -801,42 +785,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
}
|
||||
|
||||
var markersHandler *handlers.MarkersHandler
|
||||
if deps.FileRepo != nil {
|
||||
var notifier handlers.PlaybackMarkerUpdateNotifier
|
||||
if playbackHandler != nil {
|
||||
notifier = playbackHandler.MarkerUpdateNotifier
|
||||
}
|
||||
var contributor handlers.MarkerContributor
|
||||
if deps.MarkerContributionService != nil {
|
||||
contributor = deps.MarkerContributionService
|
||||
}
|
||||
var contributions handlers.MarkerContributionLister
|
||||
if deps.MarkerContributionStore != nil {
|
||||
contributions = deps.MarkerContributionStore
|
||||
}
|
||||
markersHandler = handlers.NewMarkersHandler(
|
||||
deps.FileRepo, deps.FileRepo, contributor, contributions, notifier, slog.Default(),
|
||||
)
|
||||
markersHandler.BaseContext = deps.AppContext
|
||||
markersHandler.AuditHistory = deps.FileRepo
|
||||
markersHandler.Users = userRepo
|
||||
if itemRepo != nil {
|
||||
markersHandler.Authorizer = &handlers.MediaFileAuthorizer{
|
||||
FileResolver: deps.FileRepo,
|
||||
ItemAccess: itemRepo,
|
||||
EpisodeLookup: episodeRepo,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var adminMarkerProvidersHandler *handlers.AdminMarkerProvidersHandler
|
||||
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
|
||||
adminMarkerProvidersHandler = handlers.NewAdminMarkerProvidersHandler(
|
||||
deps.MarkerRegistry, deps.MarkerProviderConfig, deps.EventBus, slog.Default(),
|
||||
)
|
||||
}
|
||||
|
||||
// Admin subtitle config handler only needs the DB repo — no S3 required.
|
||||
var adminSubtitleHandler *handlers.AdminSubtitleHandler
|
||||
var subtitleManager *subtitles.Manager
|
||||
@@ -1011,7 +959,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
libraryCollectionService.TraktTokenResolver = &traktCollectionTokenResolver{
|
||||
pool: deps.DB,
|
||||
settings: settingsRepo,
|
||||
cipher: deps.SecretCipher,
|
||||
provider: watchtrakt.NewProvider(nil, ""),
|
||||
}
|
||||
}
|
||||
@@ -1171,7 +1118,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
var historyImportHandler *handlers.HistoryImportHandler
|
||||
var historyImportSvc *historyimport.Service
|
||||
if deps.DB != nil {
|
||||
historyRepo := historyimport.NewRepository(deps.DB, deps.SecretCipher)
|
||||
historyRepo := historyimport.NewRepository(deps.DB)
|
||||
historyImportSvc = historyimport.NewService(deps.AppContext, historyRepo, deps.UserStoreProvider)
|
||||
historyIdentity := watchstate.NewStableIdentityResolver(itemRepo, episodeRepo, providerIDRepo)
|
||||
historyImportSvc.SetStableIdentityResolver(historyIdentity)
|
||||
@@ -1180,7 +1127,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
historyImportHandler = handlers.NewHistoryImportHandler(historyImportSvc)
|
||||
if deps.UserStoreProvider != nil {
|
||||
webhookSyncSvc := webhooksync.NewService(webhooksync.NewRepository(deps.DB, deps.SecretCipher), historyRepo, deps.UserStoreProvider)
|
||||
webhookSyncSvc := webhooksync.NewService(webhooksync.NewRepository(deps.DB), historyRepo, deps.UserStoreProvider)
|
||||
webhookSyncSvc.SetStableIdentityResolver(historyIdentity)
|
||||
webhookSyncHandler = handlers.NewWebhookSyncHandler(webhookSyncSvc)
|
||||
}
|
||||
@@ -1373,21 +1320,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/events/ws", eventsHandler.HandleWebSocket)
|
||||
}
|
||||
|
||||
// Marker read/write/clear for any authenticated viewer: users
|
||||
// fix and create intro/recap/credits/preview markers from the
|
||||
// player. Writes are stamped source="manual" and contributed to
|
||||
// enabled providers in the background. Contribution + provider
|
||||
// config stay admin-only (see the /admin group below).
|
||||
if markersHandler != nil {
|
||||
r.Route("/markers", func(r chi.Router) {
|
||||
r.Get("/items/{id}", markersHandler.HandleGetItemMarkers)
|
||||
r.Put("/items/{id}", markersHandler.HandleSetItemMarkers)
|
||||
r.Get("/files/{fileId}", markersHandler.HandleGetFileMarkers)
|
||||
r.Put("/files/{fileId}", markersHandler.HandleSetFileMarkers)
|
||||
r.Delete("/files/{fileId}/{segment}", markersHandler.HandleClearFileSegment)
|
||||
})
|
||||
}
|
||||
|
||||
// Library management routes (admin-only).
|
||||
if libraryHandler != nil {
|
||||
r.Group(func(r chi.Router) {
|
||||
@@ -1779,22 +1711,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
})
|
||||
}
|
||||
|
||||
if audioHandler != nil {
|
||||
r.Route("/audio", func(r chi.Router) {
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(apimw.RequireProfile)
|
||||
r.Post("/playback/start", audioHandler.HandleStartPlayback)
|
||||
r.Patch("/playback/{session_id}/sync", audioHandler.HandleSyncPlayback)
|
||||
r.Post("/playback/{session_id}/close", audioHandler.HandleClosePlayback)
|
||||
r.Get("/items/{content_id}/bookmarks", audioHandler.HandleListBookmarks)
|
||||
r.Post("/items/{content_id}/bookmarks", audioHandler.HandleCreateBookmark)
|
||||
r.Delete("/items/{content_id}/bookmarks/{time_seconds}", audioHandler.HandleDeleteBookmark)
|
||||
})
|
||||
r.Get("/playback/{session_id}/tracks/{track_index}", audioHandler.HandleStreamTrack)
|
||||
r.Head("/playback/{session_id}/tracks/{track_index}", audioHandler.HandleStreamTrack)
|
||||
})
|
||||
}
|
||||
|
||||
if watchTogetherHandler != nil {
|
||||
r.Route("/watch-together", func(r chi.Router) {
|
||||
r.Get("/rooms/{room_id}/ws", watchTogetherHandler.HandleRoomWebSocket)
|
||||
@@ -1839,6 +1755,20 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/sections/recipes", recipeHandler.HandleList)
|
||||
r.Get("/sections/recipes/{type}/candidates", recipeHandler.HandleCandidates)
|
||||
|
||||
// Audiobook endpoints (no profile required — catalog-level list/detail).
|
||||
if deps.AudiobooksEnabled && itemRepo != nil {
|
||||
audiobookHandler := &handlers.AudiobookHandler{
|
||||
Items: itemRepo,
|
||||
Files: deps.FileRepo,
|
||||
StoreProvider: deps.UserStoreProvider,
|
||||
Detail: detailSvc,
|
||||
Recs: recsRepoForStale,
|
||||
}
|
||||
r.Get("/audiobooks", audiobookHandler.HandleListAudiobooks)
|
||||
r.Get("/audiobooks/{id}", audiobookHandler.HandleGetAudiobook)
|
||||
r.Post("/audiobooks/{id}/progress", audiobookHandler.HandleReportAudiobookProgress)
|
||||
}
|
||||
|
||||
// Section endpoints (profile-scoped).
|
||||
if sectionHandler != nil {
|
||||
r.Group(func(r chi.Router) {
|
||||
@@ -1936,7 +1866,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/playback-history", adminHandler.HandleListPlaybackHistory)
|
||||
r.Get("/unmatched", adminHandler.HandleListUnmatched)
|
||||
r.Get("/stats", adminHandler.HandleGetStats)
|
||||
r.Post("/server/restart", serverControlHandler.HandleRestart)
|
||||
r.Get("/settings/sensitive-status", adminHandler.HandleGetSensitiveStatus)
|
||||
r.Post("/settings/check/{kind}", adminHandler.HandleCheckSettingsConnection)
|
||||
if sectionSettingsHandler != nil {
|
||||
@@ -1950,21 +1879,6 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Post("/items/{id}/refresh-markers", adminIntroHandler.HandleRefreshEpisodeMarkers)
|
||||
r.Post("/items/{id}/redetect-intro", adminIntroHandler.HandleRedetectEpisodeIntro)
|
||||
}
|
||||
if markersHandler != nil {
|
||||
// Marker read/write/clear live on the authenticated
|
||||
// /markers routes; writes require marker_edit.
|
||||
// Contribution and audit history stay admin operations.
|
||||
r.Post("/files/{fileId}/contribute", markersHandler.HandleContributeFile)
|
||||
r.Get("/files/{fileId}/contributions", markersHandler.HandleListFileContributions)
|
||||
r.Get("/markers/history", markersHandler.HandleListMarkerHistory)
|
||||
r.Get("/markers/files/{fileId}/history", markersHandler.HandleListFileMarkerHistory)
|
||||
r.Get("/markers/items/{id}/history", markersHandler.HandleListItemMarkerHistory)
|
||||
}
|
||||
if adminMarkerProvidersHandler != nil {
|
||||
r.Get("/markers/providers", adminMarkerProvidersHandler.HandleListProviders)
|
||||
r.Put("/markers/providers/{provider}", adminMarkerProvidersHandler.HandleUpdateProvider)
|
||||
r.Post("/markers/providers/{provider}/validate", adminMarkerProvidersHandler.HandleValidateProvider)
|
||||
}
|
||||
if peopleHandler != nil {
|
||||
r.Post("/people/{id}/refresh", peopleHandler.HandleAdminRefreshPerson)
|
||||
r.Patch("/people/{id}", peopleHandler.HandleAdminUpdatePerson)
|
||||
@@ -2130,12 +2044,10 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// System inspection.
|
||||
{
|
||||
sysJWTSecret := ""
|
||||
sysFFmpegPath := ""
|
||||
if deps.Config != nil {
|
||||
sysJWTSecret = deps.Config.Auth.JWTSecret
|
||||
sysFFmpegPath = deps.Config.Playback.FFmpegPath
|
||||
}
|
||||
systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret, sysFFmpegPath)
|
||||
systemHandler := handlers.NewSystemHandler(deps.TranscodePool, sysJWTSecret)
|
||||
r.Route("/system", func(r chi.Router) {
|
||||
r.Get("/build", systemHandler.HandleBuildInfo)
|
||||
r.Get("/hw-accel", systemHandler.HandleHWAccel)
|
||||
|
||||
@@ -11,15 +11,13 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
"github.com/Silo-Server/silo-server/internal/watchsync"
|
||||
watchtrakt "github.com/Silo-Server/silo-server/internal/watchsync/providers/trakt"
|
||||
)
|
||||
|
||||
type traktCollectionTokenResolver struct {
|
||||
pool *pgxpool.Pool
|
||||
settings catalog.SettingsStore // encrypting decorator: decrypts trakt client_secret
|
||||
cipher *secret.Cipher // decrypts watch_provider_connections tokens read via raw SQL
|
||||
settings *catalog.ServerSettingsRepo
|
||||
provider *watchtrakt.Provider
|
||||
}
|
||||
|
||||
@@ -99,7 +97,7 @@ func (r *traktCollectionTokenResolver) loadConnection(ctx context.Context, profi
|
||||
LIMIT 1
|
||||
`, profileID)
|
||||
var conn watchsync.Connection
|
||||
err := row.Scan(
|
||||
if err := row.Scan(
|
||||
&conn.ID,
|
||||
&conn.Provider,
|
||||
&conn.UserID,
|
||||
@@ -109,45 +107,24 @@ func (r *traktCollectionTokenResolver) loadConnection(ctx context.Context, profi
|
||||
&conn.AccessToken,
|
||||
&conn.RefreshToken,
|
||||
&conn.TokenExpiresAt,
|
||||
)
|
||||
if err != nil {
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return watchsync.Connection{}, errors.New("trakt connection not found for profile")
|
||||
}
|
||||
return watchsync.Connection{}, fmt.Errorf("load trakt connection: %w", err)
|
||||
}
|
||||
// This resolver reads watch_provider_connections directly (bypassing the
|
||||
// watchsync repo), so it must apply the same at-rest decryption the repo
|
||||
// does, bound to the same connection identity (watchsync.TokenAAD).
|
||||
if conn.AccessToken, err = r.cipher.DecryptIfEncrypted(conn.AccessToken, watchsync.TokenAAD("access_token", conn.Provider, conn.UserID, conn.ProfileID)); err != nil {
|
||||
return watchsync.Connection{}, fmt.Errorf("decrypt trakt access token: %w", err)
|
||||
}
|
||||
if conn.RefreshToken, err = r.cipher.DecryptIfEncrypted(conn.RefreshToken, watchsync.TokenAAD("refresh_token", conn.Provider, conn.UserID, conn.ProfileID)); err != nil {
|
||||
return watchsync.Connection{}, fmt.Errorf("decrypt trakt refresh token: %w", err)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (r *traktCollectionTokenResolver) updateTokens(ctx context.Context, conn watchsync.Connection) error {
|
||||
// Encrypt the refreshed tokens inline, bound to the connection identity, so
|
||||
// the raw-SQL write matches what the watchsync repo (and this resolver's read
|
||||
// path) store.
|
||||
accessToken, err := r.cipher.Encrypt(conn.AccessToken, watchsync.TokenAAD("access_token", conn.Provider, conn.UserID, conn.ProfileID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt trakt access token: %w", err)
|
||||
}
|
||||
refreshToken, err := r.cipher.Encrypt(conn.RefreshToken, watchsync.TokenAAD("refresh_token", conn.Provider, conn.UserID, conn.ProfileID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt trakt refresh token: %w", err)
|
||||
}
|
||||
_, err = r.pool.Exec(ctx, `
|
||||
_, err := r.pool.Exec(ctx, `
|
||||
UPDATE watch_provider_connections
|
||||
SET access_token = $2,
|
||||
refresh_token = $3,
|
||||
token_expires_at = $4,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
`, conn.ID, accessToken, refreshToken, conn.TokenExpiresAt)
|
||||
`, conn.ID, conn.AccessToken, conn.RefreshToken, conn.TokenExpiresAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update refreshed trakt connection tokens: %w", err)
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@ func (h *Handler) handleAuthorDetail(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
access, err := h.accessFilterForAuth(r.Context(), a)
|
||||
if err != nil {
|
||||
slog.Warn("abs author access resolution failed", "err", err, "id", id)
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
http.Error(w, "resolve access: "+err.Error(), http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
author, err := h.deps.MediaStore.GetAuthorByID(r.Context(), id, access)
|
||||
@@ -48,8 +47,7 @@ func (h *Handler) handleSeriesDetail(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
access, err := h.accessFilterForAuth(r.Context(), a)
|
||||
if err != nil {
|
||||
slog.Warn("abs series access resolution failed", "err", err, "id", id)
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
http.Error(w, "resolve access: "+err.Error(), http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
series, err := h.deps.MediaStore.GetSeriesByName(r.Context(), id, access)
|
||||
|
||||
@@ -44,9 +44,7 @@ func TestAuthor_Detail_ReturnsBooks(t *testing.T) {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v; body=%s", err, rec.Body.String())
|
||||
}
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &got)
|
||||
if got["name"] != "Brandon Sanderson" {
|
||||
t.Errorf("name = %v", got["name"])
|
||||
}
|
||||
@@ -80,9 +78,7 @@ func TestSeries_Detail_ReturnsBooks(t *testing.T) {
|
||||
t.Fatalf("status = %d; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v; body=%s", err, rec.Body.String())
|
||||
}
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &got)
|
||||
if got["name"] != "Mistborn" {
|
||||
t.Errorf("name = %v", got["name"])
|
||||
}
|
||||
|
||||
@@ -61,12 +61,7 @@ func (h *Handler) handleUpsertBookmark(reason string) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
item, err := h.deps.MediaStore.GetAudiobookByID(r.Context(), itemID, access)
|
||||
if err != nil {
|
||||
slog.Error("abs bookmark item lookup failed", "err", err, "user", a.UserID, "item", itemID)
|
||||
http.Error(w, "item lookup failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if item == nil {
|
||||
if err != nil || item == nil {
|
||||
http.Error(w, "item not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
@@ -158,14 +157,10 @@ func (p *recordingPublisher) snapshot() []publishedEvent {
|
||||
// handler tests can drive both the 200 and 404 branches.
|
||||
type stubMediaStore struct {
|
||||
noopMediaStore
|
||||
known map[string]*models.MediaItem // itemID → row (nil means "exists but no row needed")
|
||||
lookupErr error
|
||||
known map[string]*models.MediaItem // itemID → row (nil means "exists but no row needed")
|
||||
}
|
||||
|
||||
func (s *stubMediaStore) GetAudiobookByID(_ context.Context, id string, _ catalog.AccessFilter) (*models.MediaItem, error) {
|
||||
if s.lookupErr != nil {
|
||||
return nil, s.lookupErr
|
||||
}
|
||||
if it, ok := s.known[id]; ok {
|
||||
if it == nil {
|
||||
return &models.MediaItem{ContentID: id}, nil
|
||||
@@ -416,19 +411,6 @@ func TestMissingItem_404(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemLookup_ErrorFromMediaStore_500(t *testing.T) {
|
||||
hb := newBookmarksHarness(t, "book-1")
|
||||
hb.H.deps.MediaStore = &stubMediaStore{
|
||||
known: map[string]*models.MediaItem{"book-1": nil},
|
||||
lookupErr: errors.New("media lookup failed"),
|
||||
}
|
||||
|
||||
rec := dispatchBookmark(hb.H, http.MethodPost, "/api/me/item/book-1/bookmark", "book-1", "", []byte(`{"title":"x","time":1}`), "1", "", hb.H.handleUpsertBookmark("bookmark_created"))
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Errorf("status = %d, want 500; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidBody_400(t *testing.T) {
|
||||
hb := newBookmarksHarness(t, "book-1")
|
||||
rec := dispatchBookmark(hb.H, http.MethodPost, "/api/me/item/book-1/bookmark", "book-1", "", []byte(`{not json`), "1", "", hb.H.handleUpsertBookmark("bookmark_created"))
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/oklog/ulid/v2"
|
||||
@@ -40,12 +39,7 @@ func (h *Handler) handleCreateCollection(w http.ResponseWriter, r *http.Request)
|
||||
http.Error(w, "invalid body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if body.Name == nil {
|
||||
http.Error(w, "name required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
name := strings.TrimSpace(*body.Name)
|
||||
if name == "" {
|
||||
if body.Name == nil || *body.Name == "" {
|
||||
http.Error(w, "name required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
@@ -54,7 +48,7 @@ func (h *Handler) handleCreateCollection(w http.ResponseWriter, r *http.Request)
|
||||
ID: ulid.Make().String(),
|
||||
UserID: a.UserID,
|
||||
ProfileID: a.ProfileID,
|
||||
Name: name,
|
||||
Name: *body.Name,
|
||||
}
|
||||
if body.Description != nil {
|
||||
c.Description = *body.Description
|
||||
@@ -273,12 +267,7 @@ func (h *Handler) handleUpdateCollection(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
if body.Name != nil {
|
||||
name := strings.TrimSpace(*body.Name)
|
||||
if name == "" {
|
||||
http.Error(w, "name required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
c.Name = name
|
||||
c.Name = *body.Name
|
||||
}
|
||||
if body.Description != nil {
|
||||
c.Description = *body.Description
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
// and profile_id; we mirror that so List can filter correctly).
|
||||
type memCollectionStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]Collection // id -> row
|
||||
rows map[string]Collection // id -> row
|
||||
items map[string][]CollectionItem // collection_id -> items
|
||||
}
|
||||
|
||||
@@ -281,16 +281,8 @@ func TestCollection_List_DoesNotLeakOtherUsers(t *testing.T) {
|
||||
// User 2 lists.
|
||||
rec := dispatchABSWithParams(http.MethodGet, "/api/collections", nil, nil, "2", "", hb.H.handleListCollections)
|
||||
var env map[string]any
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
||||
t.Fatalf("decode: %v; body=%s", err, rec.Body.String())
|
||||
}
|
||||
list, ok := env["collections"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("response missing 'collections' key; body=%s", rec.Body.String())
|
||||
}
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &env)
|
||||
list, _ := env["collections"].([]any)
|
||||
if len(list) != 0 {
|
||||
t.Errorf("user 2 sees %d collections, want 0", len(list))
|
||||
}
|
||||
@@ -303,16 +295,8 @@ func TestCollection_List_ProfileIsolation(t *testing.T) {
|
||||
_ = dispatchABSWithParams(http.MethodPost, "/api/collections", nil, []byte(`{"name":"A"}`), "1", pA, hb.H.handleCreateCollection)
|
||||
rec := dispatchABSWithParams(http.MethodGet, "/api/collections", nil, nil, "1", pB, hb.H.handleListCollections)
|
||||
var env map[string]any
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
|
||||
t.Fatalf("decode: %v; body=%s", err, rec.Body.String())
|
||||
}
|
||||
list, ok := env["collections"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("response missing 'collections' key; body=%s", rec.Body.String())
|
||||
}
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &env)
|
||||
list, _ := env["collections"].([]any)
|
||||
if len(list) != 0 {
|
||||
t.Errorf("profile B sees %d collections, want 0", len(list))
|
||||
}
|
||||
|
||||
@@ -32,12 +32,7 @@ func (h *Handler) setHideFromContinue(w http.ResponseWriter, r *http.Request, hi
|
||||
return
|
||||
}
|
||||
item, err := h.deps.MediaStore.GetAudiobookByID(r.Context(), itemID, access)
|
||||
if err != nil {
|
||||
slog.Error("abs continue item lookup failed", "err", err, "user", a.UserID, "item", itemID)
|
||||
http.Error(w, "item lookup failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if item == nil {
|
||||
if err != nil || item == nil {
|
||||
http.Error(w, "item not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package abs
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -74,18 +73,3 @@ func TestContinue_UnknownItem_404(t *testing.T) {
|
||||
t.Errorf("status = %d, want 404", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinue_MediaLookupError_500(t *testing.T) {
|
||||
prog := &recordingProgressFake{}
|
||||
media := &stubMediaStore{
|
||||
known: map[string]*models.MediaItem{"book-1": nil},
|
||||
lookupErr: errors.New("media lookup failed"),
|
||||
}
|
||||
h := New(Dependencies{MediaStore: media, ProgressStore: prog})
|
||||
|
||||
rec := dispatchABSWithParams(http.MethodGet, "/api/me/progress/book-1/remove-from-continue-listening",
|
||||
map[string]string{"itemId": "book-1"}, nil, "1", "", h.handleRemoveFromContinueListening)
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Errorf("status = %d, want 500; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,18 +151,14 @@ func (h *Handler) handleDeleteItemProgress(w http.ResponseWriter, r *http.Reques
|
||||
// returns the empty progress shape so the client can store offline state
|
||||
// without raising an error.
|
||||
func (h *Handler) handleSetEpisodeProgress(w http.ResponseWriter, r *http.Request) {
|
||||
if a, ok := absAuthFrom(r); !ok || a.UserID == "" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"libraryItemId": chi.URLParam(r, "libraryItemId"),
|
||||
"episodeId": chi.URLParam(r, "episodeId"),
|
||||
"currentTime": 0.0,
|
||||
"duration": 0.0,
|
||||
"isFinished": false,
|
||||
"progress": 0.0,
|
||||
"lastUpdate": 0,
|
||||
"libraryItemId": chi.URLParam(r, "libraryItemId"),
|
||||
"episodeId": chi.URLParam(r, "episodeId"),
|
||||
"currentTime": 0.0,
|
||||
"duration": 0.0,
|
||||
"isFinished": false,
|
||||
"progress": 0.0,
|
||||
"lastUpdate": 0,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -186,8 +182,8 @@ func (h *Handler) handleEbookFile(w http.ResponseWriter, _ *http.Request) {
|
||||
// optimistic update succeeds.
|
||||
func (h *Handler) handleEbookStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"libraryItemId": chi.URLParam(r, "id"),
|
||||
"fileId": chi.URLParam(r, "fileid"),
|
||||
"libraryItemId": chi.URLParam(r, "id"),
|
||||
"fileId": chi.URLParam(r, "fileid"),
|
||||
"isSupplementary": false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,8 +2,6 @@ package audiobooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
@@ -16,9 +14,9 @@ import (
|
||||
)
|
||||
|
||||
// ABSSessionStore implements abs.TokenStore on the abs_sessions table
|
||||
// (migration 147). Each JTI is stored as a SHA-256 `token_hash`; revocation
|
||||
// sets `revoked_at`. The table stores integer user_id, so we parse the
|
||||
// string UserID from ABSToken back to int.
|
||||
// (migration 139). Each JTI is stored as the `token` column; revocation sets
|
||||
// `revoked_at`. The table stores integer user_id, so we parse the string
|
||||
// UserID from ABSToken back to int.
|
||||
type ABSSessionStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
@@ -33,12 +31,12 @@ func (s *ABSSessionStore) InsertToken(ctx context.Context, tok abs.ABSToken) err
|
||||
}
|
||||
_, err = s.Pool.Exec(ctx, `
|
||||
INSERT INTO abs_sessions
|
||||
(user_id, profile_id, token_hash, token_type, expires_at, device_id, device_name, client_name, client_version, last_seen_at)
|
||||
(user_id, profile_id, token, token_type, expires_at, device_id, device_name, client_name, client_version, last_seen_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
||||
ON CONFLICT (token_hash) DO NOTHING`,
|
||||
ON CONFLICT (token) DO NOTHING`,
|
||||
uid,
|
||||
tok.ProfileID,
|
||||
absTokenHash(tok.JTI),
|
||||
tok.JTI,
|
||||
tok.Type,
|
||||
tok.ExpiresAt,
|
||||
tok.JTI, // device_id: use the JTI as a stable device key
|
||||
@@ -52,7 +50,7 @@ func (s *ABSSessionStore) InsertToken(ctx context.Context, tok abs.ABSToken) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTokenByJTI looks up an abs_sessions row by the SHA-256 hash of its JTI.
|
||||
// GetTokenByJTI looks up an abs_sessions row by its JTI (stored in `token`).
|
||||
// Returns abs.ErrNotFound when the row doesn't exist.
|
||||
func (s *ABSSessionStore) GetTokenByJTI(ctx context.Context, jti string) (abs.ABSToken, error) {
|
||||
var (
|
||||
@@ -62,18 +60,17 @@ func (s *ABSSessionStore) GetTokenByJTI(ctx context.Context, jti string) (abs.AB
|
||||
tok abs.ABSToken
|
||||
)
|
||||
row := s.Pool.QueryRow(ctx, `
|
||||
SELECT user_id, profile_id, token_type, expires_at, revoked_at
|
||||
SELECT user_id, profile_id, token, token_type, expires_at, revoked_at
|
||||
FROM abs_sessions
|
||||
WHERE token_hash = $1`, absTokenHash(jti))
|
||||
err := row.Scan(&uid, &profileID, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
WHERE token = $1`, jti)
|
||||
err := row.Scan(&uid, &profileID, &tok.JTI, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return abs.ABSToken{}, abs.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return abs.ABSToken{}, fmt.Errorf("abs_session_store: get token: %w", err)
|
||||
}
|
||||
tok.JTI = jti
|
||||
tok.ID = jti
|
||||
tok.ID = tok.JTI
|
||||
tok.UserID = strconv.Itoa(uid)
|
||||
tok.ProfileID = profileID
|
||||
if expiresAt != nil {
|
||||
@@ -87,7 +84,7 @@ func (s *ABSSessionStore) GetTokenByJTI(ctx context.Context, jti string) (abs.AB
|
||||
func (s *ABSSessionStore) RevokeTokenByJTI(ctx context.Context, jti string) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE abs_sessions SET revoked_at = now()
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL`, absTokenHash(jti))
|
||||
WHERE token = $1 AND revoked_at IS NULL`, jti)
|
||||
if err != nil {
|
||||
return fmt.Errorf("abs_session_store: revoke token: %w", err)
|
||||
}
|
||||
@@ -104,17 +101,16 @@ func (s *ABSSessionStore) RevokeTokenIfActive(ctx context.Context, jti string) (
|
||||
row := s.Pool.QueryRow(ctx, `
|
||||
UPDATE abs_sessions
|
||||
SET revoked_at = now()
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL
|
||||
RETURNING user_id, profile_id, token_type, expires_at, revoked_at`, absTokenHash(jti))
|
||||
err := row.Scan(&uid, &profileID, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
WHERE token = $1 AND revoked_at IS NULL
|
||||
RETURNING user_id, profile_id, token, token_type, expires_at, revoked_at`, jti)
|
||||
err := row.Scan(&uid, &profileID, &tok.JTI, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return abs.ABSToken{}, abs.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return abs.ABSToken{}, fmt.Errorf("abs_session_store: revoke active token: %w", err)
|
||||
}
|
||||
tok.JTI = jti
|
||||
tok.ID = jti
|
||||
tok.ID = tok.JTI
|
||||
tok.UserID = strconv.Itoa(uid)
|
||||
tok.ProfileID = profileID
|
||||
if expiresAt != nil {
|
||||
@@ -145,14 +141,9 @@ func (s *ABSSessionStore) RevokeTokensForPrincipal(ctx context.Context, userID,
|
||||
func (s *ABSSessionStore) TouchToken(ctx context.Context, jti string) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE abs_sessions SET last_seen_at = now()
|
||||
WHERE token_hash = $1`, absTokenHash(jti))
|
||||
WHERE token = $1`, jti)
|
||||
if err != nil {
|
||||
return fmt.Errorf("abs_session_store: touch token: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func absTokenHash(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
@@ -13,20 +13,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
absJWTSecretKey = "audiobooks.abs.jwt_secret"
|
||||
absDefaultAccessTTL = 24 * time.Hour
|
||||
absDefaultRefreshTTL = 30 * 24 * time.Hour
|
||||
absJWTSecretKey = "audiobooks.abs.jwt_secret"
|
||||
absDefaultAccessTTL = 24 * time.Hour
|
||||
absDefaultRefreshTTL = 30 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// ABSConfigProvider implements abs.ConfigProvider using silo's server_settings
|
||||
// table. The ABS JWT secret is generated once on first read and persisted for
|
||||
// the lifetime of the deployment.
|
||||
type ABSConfigProvider struct {
|
||||
// Settings is the encrypting settings decorator in production, so the ABS
|
||||
// JWT secret (a SensitiveSettingKey) rests as ciphertext and is transparently
|
||||
// decrypted here. Typed as the interface so both the raw repo and the
|
||||
// decorator satisfy it.
|
||||
Settings catalog.SettingsStore
|
||||
Settings *catalog.ServerSettingsRepo
|
||||
|
||||
// secretCache holds the generated secret so we only hit the DB once per
|
||||
// process lifetime. Protected by mu.
|
||||
|
||||
@@ -21,8 +21,10 @@ type SettingsReader interface {
|
||||
GetString(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
// Service owns the Audiobookshelf compatibility adapters. Native audiobook
|
||||
// libraries are normal catalog media_items and do not depend on this service.
|
||||
// Service is the audiobooks feature's top-level orchestrator. Sub-plan 1
|
||||
// exposes only Enabled(); subsequent sub-plans hang additional methods
|
||||
// off Service as new capabilities (scanner branches, ABS handlers, etc.)
|
||||
// come online.
|
||||
type Service struct {
|
||||
settings SettingsReader
|
||||
ABSHandler *abs.Handler
|
||||
@@ -42,8 +44,8 @@ type ABSHandlerDeps struct {
|
||||
Pool *pgxpool.Pool
|
||||
Items *catalog.ItemRepository
|
||||
Files *scanner.FileRepository
|
||||
Settings catalog.SettingsStore // encrypting decorator in production
|
||||
Auth absAuthAdapter // see BuildABSHandler below
|
||||
Settings *catalog.ServerSettingsRepo
|
||||
Auth absAuthAdapter // see BuildABSHandler below
|
||||
AccessResolver abs.AccessResolver
|
||||
// Recs is the recommendations repository used to power the
|
||||
// /items/{id}/similar endpoint via embedding nearest-neighbor search.
|
||||
@@ -179,16 +181,17 @@ func buildABSRecommender(deps ABSHandlerDeps) abs.Recommender {
|
||||
return &ABSRecommender{Pool: deps.Pool, Recs: deps.Recs}
|
||||
}
|
||||
|
||||
// ABSCompatEnabled reports whether the Audiobookshelf compatibility listener
|
||||
// should be started. Native audiobook libraries intentionally do not read this
|
||||
// setting.
|
||||
func (s *Service) ABSCompatEnabled(ctx context.Context) (bool, error) {
|
||||
// Enabled reports whether the audiobooks feature flag (set by migration
|
||||
// 142 and toggled by operators) is currently true. Any value other than
|
||||
// the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
return false, nil
|
||||
}
|
||||
value, err := s.settings.GetString(ctx, "audiobookshelf_compat.enabled")
|
||||
value, err := s.settings.GetString(ctx, "audiobooks.enabled")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read audiobookshelf_compat.enabled: %w", err)
|
||||
return false, fmt.Errorf("read audiobooks.enabled: %w", err)
|
||||
}
|
||||
return value == "true", nil
|
||||
}
|
||||
|
||||
@@ -12,13 +12,13 @@ type fakeSettingsReader struct {
|
||||
}
|
||||
|
||||
func (f *fakeSettingsReader) GetString(_ context.Context, key string) (string, error) {
|
||||
if key != "audiobookshelf_compat.enabled" {
|
||||
if key != "audiobooks.enabled" {
|
||||
return "", errors.New("unexpected key: " + key)
|
||||
}
|
||||
return f.value, f.err
|
||||
}
|
||||
|
||||
func TestServiceABSCompatEnabledReadsFlag(t *testing.T) {
|
||||
func TestServiceEnabledReadsFlag(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
stored string
|
||||
@@ -33,44 +33,22 @@ func TestServiceABSCompatEnabledReadsFlag(t *testing.T) {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
svc := New(&fakeSettingsReader{value: tc.stored})
|
||||
got, err := svc.ABSCompatEnabled(context.Background())
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ABSCompatEnabled returned error: %v", err)
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("ABSCompatEnabled = %v, want %v", got, tc.want)
|
||||
t.Fatalf("Enabled = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceABSCompatEnabledPropagatesError(t *testing.T) {
|
||||
func TestServiceEnabledPropagatesError(t *testing.T) {
|
||||
wantErr := errors.New("db down")
|
||||
svc := New(&fakeSettingsReader{err: wantErr})
|
||||
_, err := svc.ABSCompatEnabled(context.Background())
|
||||
_, err := svc.Enabled(context.Background())
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("ABSCompatEnabled error = %v, want %v wrapped", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceABSCompatEnabledNilReceiverReturnsFalse(t *testing.T) {
|
||||
var svc *Service
|
||||
got, err := svc.ABSCompatEnabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ABSCompatEnabled returned error: %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatal("ABSCompatEnabled = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceABSCompatEnabledNilSettingsReturnsFalse(t *testing.T) {
|
||||
svc := New(nil)
|
||||
got, err := svc.ABSCompatEnabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ABSCompatEnabled returned error: %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatal("ABSCompatEnabled = true, want false")
|
||||
t.Fatalf("Enabled error = %v, want %v wrapped", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,13 +10,9 @@ import (
|
||||
|
||||
type Permission string
|
||||
|
||||
const (
|
||||
PermissionMarkerEdit Permission = "marker_edit"
|
||||
PermissionMetadataCuration Permission = "metadata_curation"
|
||||
)
|
||||
const PermissionMetadataCuration Permission = "metadata_curation"
|
||||
|
||||
var assignablePermissions = map[Permission]struct{}{
|
||||
PermissionMarkerEdit: {},
|
||||
PermissionMetadataCuration: {},
|
||||
}
|
||||
|
||||
@@ -60,10 +56,6 @@ func NormalizePermissions(values []string) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func DefaultUserPermissions() []string {
|
||||
return []string{string(PermissionMarkerEdit)}
|
||||
}
|
||||
|
||||
func HasAssignedPermission(user *models.User, permission Permission) bool {
|
||||
if user == nil {
|
||||
return false
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
func TestNormalizePermissions_DeduplicatesAndSorts(t *testing.T) {
|
||||
got, err := NormalizePermissions([]string{
|
||||
"marker_edit",
|
||||
" metadata_curation ",
|
||||
"metadata_curation",
|
||||
"",
|
||||
@@ -17,7 +16,7 @@ func TestNormalizePermissions_DeduplicatesAndSorts(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizePermissions returned error: %v", err)
|
||||
}
|
||||
want := []string{"marker_edit", "metadata_curation"}
|
||||
want := []string{"metadata_curation"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("permissions = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -29,14 +28,11 @@ func TestNormalizePermissions_RejectsUnknownPermission(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEffectivePermission_AdminImpliesAssignablePermissions(t *testing.T) {
|
||||
func TestHasEffectivePermission_AdminImpliesMetadataCuration(t *testing.T) {
|
||||
user := &models.User{Role: "admin", Enabled: true}
|
||||
if !HasEffectivePermission(user, PermissionMetadataCuration) {
|
||||
t.Fatal("admin should have metadata curation")
|
||||
}
|
||||
if !HasEffectivePermission(user, PermissionMarkerEdit) {
|
||||
t.Fatal("admin should have marker edit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEffectivePermission_UserRequiresAssignedPermission(t *testing.T) {
|
||||
@@ -49,11 +45,3 @@ func TestHasEffectivePermission_UserRequiresAssignedPermission(t *testing.T) {
|
||||
t.Fatal("assigned user should have metadata curation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultUserPermissionsIncludesMarkerEditOnly(t *testing.T) {
|
||||
got := DefaultUserPermissions()
|
||||
want := []string{"marker_edit"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("default permissions = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,11 +135,7 @@ func (r *UserRepository) Create(ctx context.Context, input models.CreateUserInpu
|
||||
localPasswordLoginEnabled = *input.LocalPasswordLoginEnabled
|
||||
}
|
||||
|
||||
permissions := append([]string(nil), input.Permissions...)
|
||||
if input.Permissions == nil && input.Role != "admin" {
|
||||
permissions = DefaultUserPermissions()
|
||||
}
|
||||
permissions, err = NormalizePermissions(permissions)
|
||||
permissions, err := NormalizePermissions(input.Permissions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -13,27 +13,31 @@ type ResolvedConnection struct {
|
||||
}
|
||||
|
||||
// RequestIntegrationLookup resolves a soft-linked Requests integration to its
|
||||
// base URL and (already-decrypted) api key.
|
||||
// base URL and encrypted api-key reference.
|
||||
type RequestIntegrationLookup interface {
|
||||
Get(ctx context.Context, integrationID string) (baseURL, apiKey string, err error)
|
||||
Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error)
|
||||
}
|
||||
|
||||
// SecretResolver resolves an encrypted api-key reference to its plaintext value.
|
||||
type SecretResolver interface {
|
||||
Get(ctx context.Context, ref string) (string, error)
|
||||
}
|
||||
|
||||
type ConnectionResolver struct {
|
||||
requests RequestIntegrationLookup
|
||||
secrets SecretResolver
|
||||
}
|
||||
|
||||
func NewConnectionResolver(r RequestIntegrationLookup) *ConnectionResolver {
|
||||
return &ConnectionResolver{requests: r}
|
||||
func NewConnectionResolver(r RequestIntegrationLookup, s SecretResolver) *ConnectionResolver {
|
||||
return &ConnectionResolver{requests: r, secrets: s}
|
||||
}
|
||||
|
||||
// Resolve turns a stored Connection into concrete credentials. When the
|
||||
// connection is linked to a Requests integration the live base URL + key are
|
||||
// read from there (the Requests repo decrypts it); otherwise the connection's
|
||||
// own fields are used (the autoscan repo already decrypted api_key_ref on read).
|
||||
// Either way the returned key is plaintext — there is no longer a separate
|
||||
// secret-resolution step.
|
||||
// connection is linked to a Requests integration the live base URL + key ref are
|
||||
// read from there; otherwise the connection's own fields are used. The api-key
|
||||
// ref is then resolved to plaintext via the secrets resolver.
|
||||
func (cr *ConnectionResolver) Resolve(ctx context.Context, c Connection) (ResolvedConnection, error) {
|
||||
baseURL, apiKey := c.BaseURL, c.APIKeyRef
|
||||
baseURL, apiKeyRef := c.BaseURL, c.APIKeyRef
|
||||
// A pointer-to-empty-string request_integration_id is NOT a live link (it can
|
||||
// arise from a both-NULL orphan or a stripped link). Treat empty/whitespace as
|
||||
// "no link" so we don't call requests.Get("") and use the connection's own
|
||||
@@ -42,11 +46,21 @@ func (cr *ConnectionResolver) Resolve(ctx context.Context, c Connection) (Resolv
|
||||
if cr.requests == nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: no lookup configured", *c.RequestIntegrationID)
|
||||
}
|
||||
u, key, err := cr.requests.Get(ctx, *c.RequestIntegrationID)
|
||||
u, ref, err := cr.requests.Get(ctx, *c.RequestIntegrationID)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: %w", *c.RequestIntegrationID, err)
|
||||
}
|
||||
baseURL, apiKey = u, key
|
||||
baseURL, apiKeyRef = u, ref
|
||||
}
|
||||
return ResolvedConnection{BaseURL: baseURL, APIKey: strings.TrimSpace(apiKey)}, nil
|
||||
apiKey := strings.TrimSpace(apiKeyRef)
|
||||
if cr.secrets != nil && apiKey != "" {
|
||||
resolved, err := cr.secrets.Get(ctx, apiKey)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: resolve api key: %w", err)
|
||||
}
|
||||
if resolved = strings.TrimSpace(resolved); resolved != "" {
|
||||
apiKey = resolved
|
||||
}
|
||||
}
|
||||
return ResolvedConnection{BaseURL: baseURL, APIKey: apiKey}, nil
|
||||
}
|
||||
|
||||
@@ -6,10 +6,9 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeRequestLookup implements RequestIntegrationLookup from a static map. The
|
||||
// returned apiKey is already plaintext (the Requests repo decrypts on read).
|
||||
// fakeRequestLookup implements RequestIntegrationLookup from a static map.
|
||||
type fakeRequestLookup struct {
|
||||
entries map[string]struct{ baseURL, apiKey string }
|
||||
entries map[string]struct{ baseURL, apiKeyRef string }
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -21,17 +20,30 @@ func (f fakeRequestLookup) Get(_ context.Context, id string) (string, string, er
|
||||
if !ok {
|
||||
return "", "", errors.New("integration not found: " + id)
|
||||
}
|
||||
return e.baseURL, e.apiKey, nil
|
||||
return e.baseURL, e.apiKeyRef, nil
|
||||
}
|
||||
|
||||
// fakeSecrets implements SecretResolver from a static map; unknown refs resolve
|
||||
// to empty (so the ref itself is used).
|
||||
type fakeSecrets struct {
|
||||
values map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeSecrets) Get(_ context.Context, ref string) (string, error) {
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
return f.values[ref], nil
|
||||
}
|
||||
|
||||
func TestResolveConnectionOwnCredentials(t *testing.T) {
|
||||
// The autoscan repo already decrypted api_key_ref, so the connection carries
|
||||
// the literal key; Resolve returns it verbatim (trimmed).
|
||||
r := NewConnectionResolver(fakeRequestLookup{})
|
||||
secrets := fakeSecrets{values: map[string]string{"ownref": "OWNKEY"}}
|
||||
r := NewConnectionResolver(fakeRequestLookup{}, secrets)
|
||||
|
||||
got, err := r.Resolve(context.Background(), Connection{
|
||||
BaseURL: "http://own:8989",
|
||||
APIKeyRef: "OWNKEY",
|
||||
APIKeyRef: "ownref",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve own: %v", err)
|
||||
@@ -42,10 +54,11 @@ func TestResolveConnectionOwnCredentials(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinked(t *testing.T) {
|
||||
lookup := fakeRequestLookup{entries: map[string]struct{ baseURL, apiKey string }{
|
||||
"req-1": {baseURL: "http://req:7878", apiKey: "REQKEY"},
|
||||
lookup := fakeRequestLookup{entries: map[string]struct{ baseURL, apiKeyRef string }{
|
||||
"req-1": {baseURL: "http://req:7878", apiKeyRef: "rk"},
|
||||
}}
|
||||
r := NewConnectionResolver(lookup)
|
||||
secrets := fakeSecrets{values: map[string]string{"rk": "REQKEY"}}
|
||||
r := NewConnectionResolver(lookup, secrets)
|
||||
|
||||
id := "req-1"
|
||||
got, err := r.Resolve(context.Background(), Connection{RequestIntegrationID: &id})
|
||||
@@ -58,7 +71,7 @@ func TestResolveConnectionLinked(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinkedMissingErrors(t *testing.T) {
|
||||
r := NewConnectionResolver(fakeRequestLookup{})
|
||||
r := NewConnectionResolver(fakeRequestLookup{}, fakeSecrets{})
|
||||
id := "gone"
|
||||
if _, err := r.Resolve(context.Background(), Connection{RequestIntegrationID: &id}); err == nil {
|
||||
t.Fatal("expected error when linked Requests integration is missing")
|
||||
@@ -66,7 +79,7 @@ func TestResolveConnectionLinkedMissingErrors(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinkedLookupErrors(t *testing.T) {
|
||||
r := NewConnectionResolver(fakeRequestLookup{err: errors.New("boom")})
|
||||
r := NewConnectionResolver(fakeRequestLookup{err: errors.New("boom")}, fakeSecrets{})
|
||||
id := "req-1"
|
||||
if _, err := r.Resolve(context.Background(), Connection{RequestIntegrationID: &id}); err == nil {
|
||||
t.Fatal("expected error when the lookup itself fails")
|
||||
@@ -79,12 +92,13 @@ func TestResolveConnectionEmptyIntegrationIDIsNotALink(t *testing.T) {
|
||||
// requests.Get(""). The lookup here errors on any call, so reaching it fails.
|
||||
r := NewConnectionResolver(
|
||||
fakeRequestLookup{err: errors.New("lookup must not be called")},
|
||||
fakeSecrets{values: map[string]string{"ownref": "OWNKEY"}},
|
||||
)
|
||||
for _, empty := range []string{"", " "} {
|
||||
id := empty
|
||||
got, err := r.Resolve(context.Background(), Connection{
|
||||
BaseURL: "http://own:8989",
|
||||
APIKeyRef: "OWNKEY",
|
||||
APIKeyRef: "ownref",
|
||||
RequestIntegrationID: &id,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -96,18 +110,20 @@ func TestResolveConnectionEmptyIntegrationIDIsNotALink(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionTrimsKey(t *testing.T) {
|
||||
// Resolve trims surrounding whitespace from the (decrypted) key.
|
||||
r := NewConnectionResolver(fakeRequestLookup{})
|
||||
func TestResolveConnectionTrimsAndFallsBackOnEmptySecret(t *testing.T) {
|
||||
// Whitespace-padded ref; secret resolves to whitespace-only -> fall back to
|
||||
// the trimmed ref (parity with requests.resolveAPIKey).
|
||||
secrets := fakeSecrets{values: map[string]string{"ownref": " "}}
|
||||
r := NewConnectionResolver(fakeRequestLookup{}, secrets)
|
||||
|
||||
got, err := r.Resolve(context.Background(), Connection{
|
||||
BaseURL: "http://own:8989",
|
||||
APIKeyRef: " OWNKEY ",
|
||||
APIKeyRef: " ownref ",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
if got.APIKey != "OWNKEY" {
|
||||
t.Fatalf("expected trimmed key, got %q", got.APIKey)
|
||||
if got.APIKey != "ownref" {
|
||||
t.Fatalf("expected trimmed ref fallback, got %q", got.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,17 +9,19 @@ import (
|
||||
// enriched with the metadata the Add-source picker needs (plugin id + a
|
||||
// human-friendly display name).
|
||||
type DiscoveredSource struct {
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
// PluginID is the installation's plugin id (e.g. "sonarr"); empty when the
|
||||
// lister cannot supply it.
|
||||
PluginID string
|
||||
CapabilityID string
|
||||
PluginID string
|
||||
// DisplayName is a human-friendly label for the capability (from the
|
||||
// capability's manifest display_name, falling back to plugin/capability ids).
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
// ScanSourceLister enumerates every installed scan_source.v1 capability so the
|
||||
// engine can offer them in the Add-source picker.
|
||||
// engine can (a) offer them in the Add-source picker and (b) detect orphaned
|
||||
// source rows whose plugin has been uninstalled.
|
||||
type ScanSourceLister interface {
|
||||
// ListScanSources returns one entry per installed scan_source.v1 capability,
|
||||
// enriched with plugin id + display name.
|
||||
@@ -29,9 +31,17 @@ type ScanSourceLister interface {
|
||||
// AvailableScanSource is one installed scan_source capability an operator can
|
||||
// create a source against (the Add-source picker list).
|
||||
type AvailableScanSource struct {
|
||||
PluginID string `json:"plugin_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
// installedKey identifies an installed scan_source capability for set
|
||||
// membership tests (orphan detection in PollOnce).
|
||||
type installedKey struct {
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
}
|
||||
|
||||
// ListAvailableScanSources enumerates every installed scan_source capability so
|
||||
@@ -49,10 +59,30 @@ func (s *Service) ListAvailableScanSources(ctx context.Context) ([]AvailableScan
|
||||
out := make([]AvailableScanSource, 0, len(discovered))
|
||||
for _, d := range discovered {
|
||||
out = append(out, AvailableScanSource{
|
||||
PluginID: d.PluginID,
|
||||
CapabilityID: d.CapabilityID,
|
||||
DisplayName: d.DisplayName,
|
||||
InstallationID: d.InstallationID,
|
||||
CapabilityID: d.CapabilityID,
|
||||
PluginID: d.PluginID,
|
||||
DisplayName: d.DisplayName,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// installedScanSources returns the set of currently-installed scan_source
|
||||
// capabilities so PollOnce can skip orphaned source rows (their plugin is
|
||||
// gone). A nil set means the set is unavailable (no lister) — callers must treat
|
||||
// that as "discovery unavailable" and NOT prune.
|
||||
func (s *Service) installedScanSources(ctx context.Context) (map[installedKey]struct{}, error) {
|
||||
if s.lister == nil {
|
||||
return nil, nil
|
||||
}
|
||||
discovered, err := s.lister.ListScanSources(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list scan sources: %w", err)
|
||||
}
|
||||
present := make(map[installedKey]struct{}, len(discovered))
|
||||
for _, d := range discovered {
|
||||
present[installedKey{InstallationID: d.InstallationID, CapabilityID: d.CapabilityID}] = struct{}{}
|
||||
}
|
||||
return present, nil
|
||||
}
|
||||
|
||||
@@ -14,8 +14,8 @@ func (f fakeLister) ListScanSources(context.Context) ([]DiscoveredSource, error)
|
||||
|
||||
func TestListAvailableScanSourcesEnumeratesInstalled(t *testing.T) {
|
||||
lister := fakeLister{sources: []DiscoveredSource{
|
||||
{PluginID: "sonarr", CapabilityID: "arr-a", DisplayName: "Sonarr"},
|
||||
{PluginID: "radarr", CapabilityID: "arr-b", DisplayName: "Radarr"},
|
||||
{InstallationID: 1, CapabilityID: "arr-a", PluginID: "sonarr", DisplayName: "Sonarr"},
|
||||
{InstallationID: 2, CapabilityID: "arr-b", PluginID: "radarr", DisplayName: "Radarr"},
|
||||
}}
|
||||
svc := &Service{lister: lister}
|
||||
|
||||
@@ -26,7 +26,7 @@ func TestListAvailableScanSourcesEnumeratesInstalled(t *testing.T) {
|
||||
if len(available) != 2 {
|
||||
t.Fatalf("expected 2 available, got %d: %+v", len(available), available)
|
||||
}
|
||||
if available[0] != (AvailableScanSource{PluginID: "sonarr", CapabilityID: "arr-a", DisplayName: "Sonarr"}) {
|
||||
if available[0] != (AvailableScanSource{InstallationID: 1, CapabilityID: "arr-a", PluginID: "sonarr", DisplayName: "Sonarr"}) {
|
||||
t.Fatalf("unexpected first available: %+v", available[0])
|
||||
}
|
||||
}
|
||||
@@ -41,3 +41,32 @@ func TestListAvailableScanSourcesNilListerEmpty(t *testing.T) {
|
||||
t.Fatalf("nil lister must return empty, got %+v", available)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledScanSourcesSetMembership(t *testing.T) {
|
||||
lister := fakeLister{sources: []DiscoveredSource{
|
||||
{InstallationID: 1, CapabilityID: "arr-a"},
|
||||
}}
|
||||
svc := &Service{lister: lister}
|
||||
|
||||
present, err := svc.installedScanSources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("installedScanSources: %v", err)
|
||||
}
|
||||
if _, ok := present[installedKey{1, "arr-a"}]; !ok {
|
||||
t.Fatalf("expected 1/arr-a present: %+v", present)
|
||||
}
|
||||
if _, ok := present[installedKey{2, "arr-b"}]; ok {
|
||||
t.Fatalf("did not expect 2/arr-b present: %+v", present)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledScanSourcesNilListerNilSet(t *testing.T) {
|
||||
svc := &Service{lister: nil}
|
||||
present, err := svc.installedScanSources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("installedScanSources: %v", err)
|
||||
}
|
||||
if present != nil {
|
||||
t.Fatalf("nil lister must return a nil set, got %+v", present)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,3 @@ import "errors"
|
||||
// ErrNotFound is returned when an autoscan operation references a connection or
|
||||
// source that does not exist. The admin API surfaces it as a 404.
|
||||
var ErrNotFound = errors.New("autoscan: not found")
|
||||
|
||||
// ErrPollAlreadyRunning is returned when a source already has an active poll
|
||||
// event. Callers should treat it as a no-op skip, not a source failure.
|
||||
var ErrPollAlreadyRunning = errors.New("autoscan: source poll already running")
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestTestConnectionNoProbeConfigured(t *testing.T) {
|
||||
func TestSuggestRewritesMatchesArrRootsToSiloFolders(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
sources: []Source{
|
||||
{ID: "src-1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1")},
|
||||
{ID: "src-1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1")},
|
||||
},
|
||||
}
|
||||
arr := &fakeArrClient{roots: []string{"/data/tv"}}
|
||||
@@ -87,7 +87,7 @@ func TestSuggestRewritesMatchesArrRootsToSiloFolders(t *testing.T) {
|
||||
|
||||
func TestSuggestRewritesNoConnection(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
sources: []Source{{ID: "src-1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: nil}},
|
||||
sources: []Source{{ID: "src-1", InstallationID: 1, CapabilityID: "arr", ConnectionID: nil}},
|
||||
}
|
||||
svc := &Service{store: store, connres: passthroughConnRes{}, rootFolders: &fakeArrClient{}, folders: fakeFolderLister{}}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
// ScanSourceProvider yields changed paths for one source. The engine calls
|
||||
// PollChanges; production wraps the plugins.Service scan_source resolver.
|
||||
type ScanSourceProvider interface {
|
||||
PollChanges(ctx context.Context, pluginID, capabilityID, marker string, conn ResolvedConnection, sourceConfig map[string]string) (changes []Change, nextMarker string, err error)
|
||||
PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection, sourceConfig map[string]string) (changes []Change, nextMarker string, err error)
|
||||
}
|
||||
|
||||
// PollChangesClient is the slice of *pluginhost.ScanSourceClient used here. It
|
||||
@@ -20,10 +20,10 @@ type PollChangesClient interface {
|
||||
PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error)
|
||||
}
|
||||
|
||||
// ScanSourceResolver yields a per-(plugin, capability) scan-source client.
|
||||
// ScanSourceResolver yields a per-(installation, capability) scan-source client.
|
||||
// Exported for the same cross-package adapter reason as PollChangesClient.
|
||||
type ScanSourceResolver interface {
|
||||
ScanSourceClient(ctx context.Context, pluginID, capabilityID string) (PollChangesClient, error)
|
||||
ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (PollChangesClient, error)
|
||||
}
|
||||
|
||||
type pluginProvider struct{ resolver ScanSourceResolver }
|
||||
@@ -34,8 +34,8 @@ func NewPluginProvider(resolver ScanSourceResolver) ScanSourceProvider {
|
||||
return &pluginProvider{resolver: resolver}
|
||||
}
|
||||
|
||||
func (p *pluginProvider) PollChanges(ctx context.Context, pluginID, capabilityID, marker string, conn ResolvedConnection, sourceConfig map[string]string) ([]Change, string, error) {
|
||||
client, err := p.resolver.ScanSourceClient(ctx, pluginID, capabilityID)
|
||||
func (p *pluginProvider) PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection, sourceConfig map[string]string) ([]Change, string, error) {
|
||||
client, err := p.resolver.ScanSourceClient(ctx, installationID, capabilityID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
@@ -17,17 +17,10 @@ func (c *capturingClient) PollChanges(_ context.Context, req *pluginv1.PollChang
|
||||
return &pluginv1.PollChangesResponse{SourcePaths: []string{"/mnt/media/x"}, NextMarker: "m2"}, nil
|
||||
}
|
||||
|
||||
// capturingResolver yields a fixed PollChangesClient and records the requested
|
||||
// stable scan-source identity.
|
||||
type capturingResolver struct {
|
||||
client PollChangesClient
|
||||
lastPluginID string
|
||||
lastCapID string
|
||||
}
|
||||
// capturingResolver yields a fixed PollChangesClient.
|
||||
type capturingResolver struct{ client PollChangesClient }
|
||||
|
||||
func (r *capturingResolver) ScanSourceClient(_ context.Context, pluginID, capabilityID string) (PollChangesClient, error) {
|
||||
r.lastPluginID = pluginID
|
||||
r.lastCapID = capabilityID
|
||||
func (r capturingResolver) ScanSourceClient(context.Context, int, string) (PollChangesClient, error) {
|
||||
return r.client, nil
|
||||
}
|
||||
|
||||
@@ -35,12 +28,11 @@ func (r *capturingResolver) ScanSourceClient(_ context.Context, pluginID, capabi
|
||||
// delivered to the plugin on the PollChangesRequest.
|
||||
func TestPluginProviderPopulatesConnection(t *testing.T) {
|
||||
client := &capturingClient{}
|
||||
resolver := &capturingResolver{client: client}
|
||||
prov := NewPluginProvider(resolver)
|
||||
prov := NewPluginProvider(capturingResolver{client})
|
||||
|
||||
conn := ResolvedConnection{BaseURL: "https://arr.example", APIKey: "secret-key"}
|
||||
sourceConfig := map[string]string{"exclusions": ".downloads"}
|
||||
changes, next, err := prov.PollChanges(context.Background(), "silo.autoscan.arr", "cap", "m1", conn, sourceConfig)
|
||||
changes, next, err := prov.PollChanges(context.Background(), 1, "cap", "m1", conn, sourceConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("PollChanges: %v", err)
|
||||
}
|
||||
@@ -53,9 +45,6 @@ func TestPluginProviderPopulatesConnection(t *testing.T) {
|
||||
if client.last.GetCapabilityId() != "cap" || client.last.GetMarker() != "m1" {
|
||||
t.Fatalf("unexpected request fields: cap=%q marker=%q", client.last.GetCapabilityId(), client.last.GetMarker())
|
||||
}
|
||||
if resolver.lastPluginID != "silo.autoscan.arr" || resolver.lastCapID != "cap" {
|
||||
t.Fatalf("resolver identity = %q/%q", resolver.lastPluginID, resolver.lastCapID)
|
||||
}
|
||||
rc := client.last.GetConnection()
|
||||
if rc == nil {
|
||||
t.Fatal("expected connection to be populated on the request")
|
||||
@@ -70,9 +59,9 @@ func TestPluginProviderPopulatesConnection(t *testing.T) {
|
||||
|
||||
func TestPluginProviderPrefersStructuredChanges(t *testing.T) {
|
||||
client := &capturingStructuredClient{}
|
||||
prov := NewPluginProvider(&capturingResolver{client: client})
|
||||
prov := NewPluginProvider(capturingResolver{client: client})
|
||||
|
||||
changes, next, err := prov.PollChanges(context.Background(), "silo.autoscan.arr", "cap", "m1", ResolvedConnection{}, nil)
|
||||
changes, next, err := prov.PollChanges(context.Background(), 1, "cap", "m1", ResolvedConnection{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("PollChanges: %v", err)
|
||||
}
|
||||
|
||||
+37
-181
@@ -9,38 +9,14 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
)
|
||||
|
||||
type Repository struct {
|
||||
pool *pgxpool.Pool
|
||||
cipher *secret.Cipher
|
||||
}
|
||||
type Repository struct{ pool *pgxpool.Pool }
|
||||
|
||||
func NewRepository(pool *pgxpool.Pool, cipher *secret.Cipher) *Repository {
|
||||
return &Repository{pool: pool, cipher: cipher}
|
||||
}
|
||||
|
||||
// connectionAPIKeyAAD binds an autoscan_connections api_key_ref ciphertext to
|
||||
// its row id.
|
||||
func connectionAPIKeyAAD(id string) string {
|
||||
return secret.RowAAD("autoscan_connections", "api_key_ref", id)
|
||||
}
|
||||
|
||||
// encryptAPIKey encrypts a non-empty, trimmed key bound to the connection id;
|
||||
// an empty key returns "" so NULL/keep-existing semantics are preserved.
|
||||
func (r *Repository) encryptAPIKey(id, apiKey string) (string, error) {
|
||||
apiKey = strings.TrimSpace(apiKey)
|
||||
if apiKey == "" {
|
||||
return "", nil
|
||||
}
|
||||
return r.cipher.Encrypt(apiKey, connectionAPIKeyAAD(id))
|
||||
}
|
||||
func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} }
|
||||
|
||||
// --- Settings ---
|
||||
|
||||
@@ -88,7 +64,7 @@ func (r *Repository) UpdateSettings(ctx context.Context, s Settings) (Settings,
|
||||
|
||||
const connectionColumns = `id, name, kind, base_url, api_key_ref, request_integration_id`
|
||||
|
||||
func (r *Repository) scanConnection(row interface{ Scan(...any) error }) (Connection, error) {
|
||||
func scanConnection(row interface{ Scan(...any) error }) (Connection, error) {
|
||||
var c Connection
|
||||
var baseURL, apiKeyRef, reqIntegrationID *string
|
||||
if err := row.Scan(&c.ID, &c.Name, &c.Kind, &baseURL, &apiKeyRef, &reqIntegrationID); err != nil {
|
||||
@@ -101,13 +77,6 @@ func (r *Repository) scanConnection(row interface{ Scan(...any) error }) (Connec
|
||||
c.APIKeyRef = *apiKeyRef
|
||||
}
|
||||
c.RequestIntegrationID = reqIntegrationID
|
||||
// Decrypt the stored key (read-path contract): legacy plaintext passes
|
||||
// through, enc:v1: decrypts, corrupt ciphertext errors.
|
||||
apiKey, err := r.cipher.DecryptIfEncrypted(c.APIKeyRef, connectionAPIKeyAAD(c.ID))
|
||||
if err != nil {
|
||||
return Connection{}, fmt.Errorf("decrypt autoscan connection %s api key: %w", c.ID, err)
|
||||
}
|
||||
c.APIKeyRef = apiKey
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -120,22 +89,12 @@ func nullable(s string) any {
|
||||
}
|
||||
|
||||
func (r *Repository) CreateConnection(ctx context.Context, c Connection) (Connection, error) {
|
||||
// Generate the id in Go (rather than the DB default) so the api_key_ref
|
||||
// ciphertext can be AAD-bound to the row id at insert time.
|
||||
id := strings.TrimSpace(c.ID)
|
||||
if id == "" {
|
||||
id = uuid.NewString()
|
||||
}
|
||||
apiKeyRef, err := r.encryptAPIKey(id, c.APIKeyRef)
|
||||
if err != nil {
|
||||
return Connection{}, fmt.Errorf("encrypt autoscan api key: %w", err)
|
||||
}
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_connections (id, name, kind, base_url, api_key_ref, request_integration_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
INSERT INTO autoscan_connections (name, kind, base_url, api_key_ref, request_integration_id)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING `+connectionColumns,
|
||||
id, c.Name, c.Kind, nullable(c.BaseURL), nullable(apiKeyRef), c.RequestIntegrationID)
|
||||
out, err := r.scanConnection(row)
|
||||
c.Name, c.Kind, nullable(c.BaseURL), nullable(c.APIKeyRef), c.RequestIntegrationID)
|
||||
out, err := scanConnection(row)
|
||||
if err != nil {
|
||||
return Connection{}, fmt.Errorf("create autoscan connection: %w", err)
|
||||
}
|
||||
@@ -149,11 +108,6 @@ func (r *Repository) UpdateConnection(ctx context.Context, c Connection) (Connec
|
||||
// next poll. Mirrors requests.UpdateIntegration's CASE-WHEN keep-semantics.
|
||||
// Pass the raw trimmed string (not nullable()) so the empty-string sentinel
|
||||
// reaches the CASE.
|
||||
// Encrypt the incoming key; an empty result preserves the keep-existing CASE.
|
||||
apiKeyRef, err := r.encryptAPIKey(c.ID, c.APIKeyRef)
|
||||
if err != nil {
|
||||
return Connection{}, fmt.Errorf("encrypt autoscan api key: %w", err)
|
||||
}
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
UPDATE autoscan_connections
|
||||
SET name = $2, kind = $3, base_url = $4,
|
||||
@@ -161,8 +115,8 @@ func (r *Repository) UpdateConnection(ctx context.Context, c Connection) (Connec
|
||||
request_integration_id = $6, updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING `+connectionColumns,
|
||||
c.ID, c.Name, c.Kind, nullable(c.BaseURL), apiKeyRef, c.RequestIntegrationID)
|
||||
out, err := r.scanConnection(row)
|
||||
c.ID, c.Name, c.Kind, nullable(c.BaseURL), c.APIKeyRef, c.RequestIntegrationID)
|
||||
out, err := scanConnection(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Connection{}, fmt.Errorf("%w: connection %s", ErrNotFound, c.ID)
|
||||
@@ -197,7 +151,7 @@ func (r *Repository) ListConnections(ctx context.Context) ([]Connection, error)
|
||||
defer rows.Close()
|
||||
var out []Connection
|
||||
for rows.Next() {
|
||||
c, err := r.scanConnection(rows)
|
||||
c, err := scanConnection(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -209,7 +163,7 @@ func (r *Repository) ListConnections(ctx context.Context) ([]Connection, error)
|
||||
func (r *Repository) GetConnection(ctx context.Context, id string) (Connection, error) {
|
||||
row := r.pool.QueryRow(ctx, `SELECT `+connectionColumns+`
|
||||
FROM autoscan_connections WHERE id = $1`, id)
|
||||
c, err := r.scanConnection(row)
|
||||
c, err := scanConnection(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Connection{}, fmt.Errorf("%w: connection %s", ErrNotFound, id)
|
||||
@@ -221,14 +175,14 @@ func (r *Repository) GetConnection(ctx context.Context, id string) (Connection,
|
||||
|
||||
// --- Sources ---
|
||||
|
||||
const sourceColumns = `id, plugin_id, capability_id, connection_id, enabled,
|
||||
const sourceColumns = `id, installation_id, capability_id, connection_id, enabled,
|
||||
poll_interval_seconds, path_rewrites, source_config, label, marker, last_run_at, last_error`
|
||||
|
||||
func scanSource(row interface{ Scan(...any) error }) (Source, error) {
|
||||
var s Source
|
||||
var pathRewrites []byte
|
||||
var sourceConfig []byte
|
||||
if err := row.Scan(&s.ID, &s.PluginID, &s.CapabilityID, &s.ConnectionID,
|
||||
if err := row.Scan(&s.ID, &s.InstallationID, &s.CapabilityID, &s.ConnectionID,
|
||||
&s.Enabled, &s.PollIntervalSeconds, &pathRewrites, &sourceConfig, &s.Label, &s.Marker, &s.LastRunAt, &s.LastError); err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
@@ -329,11 +283,11 @@ func (r *Repository) CreateSource(ctx context.Context, s Source) (Source, error)
|
||||
}
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_sources (
|
||||
plugin_id, capability_id, connection_id, enabled, poll_interval_seconds, path_rewrites, source_config, label
|
||||
installation_id, capability_id, connection_id, enabled, poll_interval_seconds, path_rewrites, source_config, label
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING `+sourceColumns,
|
||||
s.PluginID, s.CapabilityID, connectionIDArg(s.ConnectionID), s.Enabled, s.PollIntervalSeconds, rewrites, sourceConfig, s.Label)
|
||||
s.InstallationID, s.CapabilityID, connectionIDArg(s.ConnectionID), s.Enabled, s.PollIntervalSeconds, rewrites, sourceConfig, s.Label)
|
||||
out, err := scanSource(row)
|
||||
if err != nil {
|
||||
if connID, ok := connectionFKViolation(err, s.ConnectionID); ok {
|
||||
@@ -345,7 +299,7 @@ func (r *Repository) CreateSource(ctx context.Context, s Source) (Source, error)
|
||||
}
|
||||
|
||||
// UpdateSource updates a source's binding/scheduling fields by id. Identity
|
||||
// (plugin_id, capability_id) and bookkeeping fields (marker/last_run_at/
|
||||
// (installation_id, capability_id) and bookkeeping fields (marker/last_run_at/
|
||||
// last_error) are left untouched. An unknown id maps to ErrNotFound; a
|
||||
// non-existent connection trips the FK constraint and also maps to ErrNotFound.
|
||||
func (r *Repository) UpdateSource(ctx context.Context, s Source) (Source, error) {
|
||||
@@ -399,7 +353,7 @@ func connectionFKViolation(err error, connectionID *string) (string, bool) {
|
||||
|
||||
func (r *Repository) ListSources(ctx context.Context) ([]Source, error) {
|
||||
rows, err := r.pool.Query(ctx, `SELECT `+sourceColumns+`
|
||||
FROM autoscan_sources ORDER BY plugin_id, capability_id`)
|
||||
FROM autoscan_sources ORDER BY installation_id, capability_id`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list autoscan sources: %w", err)
|
||||
}
|
||||
@@ -410,7 +364,7 @@ func (r *Repository) ListSources(ctx context.Context) ([]Source, error) {
|
||||
func (r *Repository) ListEnabledSources(ctx context.Context) ([]Source, error) {
|
||||
rows, err := r.pool.Query(ctx, `SELECT `+sourceColumns+`
|
||||
FROM autoscan_sources WHERE enabled = true
|
||||
ORDER BY plugin_id, capability_id`)
|
||||
ORDER BY installation_id, capability_id`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list enabled autoscan sources: %w", err)
|
||||
}
|
||||
@@ -508,7 +462,7 @@ func (r *Repository) RecordError(ctx context.Context, sourceID, msg string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
const eventColumns = `id, source_id, plugin_id, capability_id, started_at, completed_at,
|
||||
const eventColumns = `id, source_id, installation_id, capability_id, started_at, completed_at,
|
||||
duration_ms, status, changes_returned, changes_resolved, targets_claimed, scans_created,
|
||||
scans_reused, scans_suppressed, error_message, marker_before, marker_after`
|
||||
|
||||
@@ -518,7 +472,7 @@ func scanEvent(row interface{ Scan(...any) error }) (Event, error) {
|
||||
if err := row.Scan(
|
||||
&e.ID,
|
||||
&e.SourceID,
|
||||
&e.PluginID,
|
||||
&e.InstallationID,
|
||||
&e.CapabilityID,
|
||||
&e.StartedAt,
|
||||
&e.CompletedAt,
|
||||
@@ -545,80 +499,25 @@ func (r *Repository) CreateEvent(ctx context.Context, in EventCreate) (int64, er
|
||||
if started.IsZero() {
|
||||
started = time.Now()
|
||||
}
|
||||
sourceID := strings.TrimSpace(in.SourceID)
|
||||
if sourceID == "" {
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_events (
|
||||
source_id, plugin_id, capability_id, started_at, completed_at,
|
||||
duration_ms, status, error_message, marker_before
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $4, 0, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
nil,
|
||||
in.PluginID,
|
||||
in.CapabilityID,
|
||||
started,
|
||||
string(EventStatusRunning),
|
||||
"",
|
||||
nullable(in.MarkerBefore),
|
||||
)
|
||||
var id int64
|
||||
if err := row.Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("create autoscan event: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("begin autoscan event: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(900173, hashtext($1))`, sourceID); err != nil {
|
||||
return 0, fmt.Errorf("lock autoscan event: %w", err)
|
||||
}
|
||||
|
||||
var runningID int64
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT id
|
||||
FROM autoscan_events
|
||||
WHERE source_id = $1
|
||||
AND status = $2
|
||||
ORDER BY started_at ASC, id ASC
|
||||
LIMIT 1`,
|
||||
sourceID,
|
||||
string(EventStatusRunning),
|
||||
).Scan(&runningID)
|
||||
if err == nil {
|
||||
return 0, fmt.Errorf("%w: source %s event %d", ErrPollAlreadyRunning, sourceID, runningID)
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, fmt.Errorf("check running autoscan event: %w", err)
|
||||
}
|
||||
|
||||
row := tx.QueryRow(ctx, `
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_events (
|
||||
source_id, plugin_id, capability_id, started_at, completed_at,
|
||||
source_id, installation_id, capability_id, started_at, completed_at,
|
||||
duration_ms, status, error_message, marker_before
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $4, 0, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
sourceID,
|
||||
in.PluginID,
|
||||
nullable(in.SourceID),
|
||||
in.InstallationID,
|
||||
in.CapabilityID,
|
||||
started,
|
||||
string(EventStatusRunning),
|
||||
"",
|
||||
string(EventStatusError),
|
||||
"poll started but did not finish",
|
||||
nullable(in.MarkerBefore),
|
||||
)
|
||||
var id int64
|
||||
if err := row.Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("create autoscan event: %w", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, fmt.Errorf("commit autoscan event: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
@@ -666,25 +565,6 @@ func (r *Repository) FinishEvent(ctx context.Context, in EventFinish) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Repository) MarkInterruptedEvents(ctx context.Context) error {
|
||||
msg := "poll started but did not finish"
|
||||
_, err := r.pool.Exec(ctx, `
|
||||
UPDATE autoscan_events
|
||||
SET completed_at = now(),
|
||||
duration_ms = GREATEST(0, EXTRACT(EPOCH FROM (now() - started_at)) * 1000)::bigint,
|
||||
status = $1,
|
||||
error_message = $2
|
||||
WHERE status = $3`,
|
||||
string(EventStatusError),
|
||||
msg,
|
||||
string(EventStatusRunning),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark interrupted autoscan events: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clampAutoscanLimit bounds a requested page size to a sane window: the
|
||||
// default page when unset, capped so a single query can never fan out.
|
||||
func clampAutoscanLimit(limit int) int {
|
||||
@@ -878,30 +758,6 @@ func (r *Repository) ListEvents(ctx context.Context, filter EventListFilter) ([]
|
||||
return events, runRows.Err()
|
||||
}
|
||||
|
||||
func (r *Repository) ListRunningEvents(ctx context.Context) ([]Event, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT `+eventColumns+`
|
||||
FROM autoscan_events
|
||||
WHERE status = $1
|
||||
ORDER BY started_at ASC, id ASC`,
|
||||
string(EventStatusRunning),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list running autoscan events: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
events := make([]Event, 0)
|
||||
for rows.Next() {
|
||||
event, scanErr := scanEvent(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
return events, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Repository) ListAutoscanScans(ctx context.Context, filter ScanListFilter) ([]ScanWithEvent, error) {
|
||||
limit := clampAutoscanLimit(filter.Limit)
|
||||
offset := filter.Offset
|
||||
@@ -924,15 +780,15 @@ func (r *Repository) ListAutoscanScans(ctx context.Context, filter ScanListFilte
|
||||
sr.trigger,
|
||||
sr.status,
|
||||
COALESCE(sr.error_message, ''),
|
||||
sr.requested_at,
|
||||
sr.started_at,
|
||||
sr.completed_at,
|
||||
sr.autoscan_event_id,
|
||||
e.source_id,
|
||||
COALESCE(e.plugin_id, ''),
|
||||
COALESCE(e.capability_id, ''),
|
||||
COALESCE(e.status, ''),
|
||||
e.completed_at
|
||||
sr.requested_at,
|
||||
sr.started_at,
|
||||
sr.completed_at,
|
||||
sr.autoscan_event_id,
|
||||
e.source_id,
|
||||
e.installation_id,
|
||||
COALESCE(e.capability_id, ''),
|
||||
COALESCE(e.status, ''),
|
||||
e.completed_at
|
||||
FROM scan_runs sr
|
||||
LEFT JOIN autoscan_events e ON e.id = sr.autoscan_event_id
|
||||
WHERE `+strings.Join(clauses, " AND ")+`
|
||||
@@ -962,7 +818,7 @@ func (r *Repository) ListAutoscanScans(ctx context.Context, filter ScanListFilte
|
||||
&scan.CompletedAt,
|
||||
&scan.AutoscanEventID,
|
||||
&scan.SourceID,
|
||||
&scan.PluginID,
|
||||
&scan.InstallationID,
|
||||
&scan.CapabilityID,
|
||||
&eventStatus,
|
||||
&scan.EventCompletedAt,
|
||||
@@ -997,7 +853,7 @@ func (r *Repository) GetQueueSummary(ctx context.Context) (QueueSummary, error)
|
||||
|
||||
func (r *Repository) LatestEventAt(ctx context.Context) (*time.Time, error) {
|
||||
var latest *time.Time
|
||||
err := r.pool.QueryRow(ctx, `SELECT max(completed_at) FROM autoscan_events WHERE status <> $1`, string(EventStatusRunning)).Scan(&latest)
|
||||
err := r.pool.QueryRow(ctx, `SELECT max(completed_at) FROM autoscan_events`).Scan(&latest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get latest autoscan event time: %w", err)
|
||||
}
|
||||
|
||||
@@ -91,7 +91,8 @@ func (s *Service) SetSuggesterDeps(rootFolders RootFolderClient, folders FolderL
|
||||
// from scan_source plugins; connres resolves a source's connection to concrete
|
||||
// credentials; resolver/queue/suppress drive the resolve→suppress→enqueue loop.
|
||||
// lister enumerates installed scan_source capabilities so the Add-source picker
|
||||
// can offer them; it may be nil (the picker then returns an empty list).
|
||||
// can offer them and PollOnce can skip orphaned source rows; it may be nil
|
||||
// (the picker then returns an empty list and orphan-pruning is disabled).
|
||||
func NewService(
|
||||
store Store,
|
||||
provider ScanSourceProvider,
|
||||
@@ -112,15 +113,24 @@ func NewService(
|
||||
}
|
||||
}
|
||||
|
||||
// PollOnce runs one autoscan cycle. Per-source failures are logged, recorded on
|
||||
// the source/event, and the loop continues; only settings/listing errors
|
||||
// propagate. The opaque next marker returned by the
|
||||
// PollOnce runs one autoscan cycle. Per-source failures are logged and skipped;
|
||||
// only settings/listing errors propagate. The opaque next marker returned by the
|
||||
// provider is stored verbatim, but only when the cycle's work is genuinely
|
||||
// consumed: the provider returned no paths, or it returned paths and at least one
|
||||
// resolved+enqueued. When paths come back but NONE resolve to a library folder
|
||||
// (e.g. a freshly-enabled source with unconfigured rewrites) the marker is held
|
||||
// and an error recorded, so those imports aren't skipped forever.
|
||||
func (s *Service) PollOnce(ctx context.Context) error {
|
||||
// Fetch the set of currently-installed scan_source capabilities so we can skip
|
||||
// orphaned source rows (their plugin was uninstalled). Listing failures are
|
||||
// non-fatal: a nil set means the installed set is unavailable, in which case
|
||||
// we do NOT prune orphans — every enabled source is assumed present.
|
||||
present, derr := s.installedScanSources(ctx)
|
||||
if derr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: list installed scan sources failed", "err", derr)
|
||||
present = nil
|
||||
}
|
||||
|
||||
settings, err := s.store.GetSettings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -136,6 +146,16 @@ func (s *Service) PollOnce(ctx context.Context) error {
|
||||
now := time.Now()
|
||||
|
||||
for _, src := range sources {
|
||||
// Skip orphaned sources: an enabled source whose scan_source plugin has
|
||||
// been uninstalled/disabled is no longer in the discovered set, so polling
|
||||
// it would error every cycle. Skip it quietly (no RecordError) so the spam
|
||||
// stops; the operator can delete it via the source delete endpoint. A nil
|
||||
// `present` set means discovery is unavailable — don't prune in that case.
|
||||
if present != nil {
|
||||
if _, ok := present[installedKey{InstallationID: src.InstallationID, CapabilityID: src.CapabilityID}]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Honor the per-source poll interval as a "poll at most every N seconds"
|
||||
// floor: the global task fires at the default cadence, so a source with a
|
||||
// longer interval is skipped until enough time has elapsed.
|
||||
@@ -150,10 +170,7 @@ func (s *Service) PollOnce(ctx context.Context) error {
|
||||
if src.Marker != nil {
|
||||
marker = *src.Marker
|
||||
}
|
||||
eventID, started := s.createEvent(ctx, src, marker, time.Now())
|
||||
if !started {
|
||||
continue
|
||||
}
|
||||
eventID := s.createEvent(ctx, src, marker, time.Now())
|
||||
// A connection is OPTIONAL. Server-based providers (Sonarr/Radarr) bind a
|
||||
// connection and the resolved {base_url, api_key} is handed to the plugin.
|
||||
// Other providers (e.g. a filesystem/CephFS watcher) need none and get an
|
||||
@@ -178,7 +195,7 @@ func (s *Service) PollOnce(ctx context.Context) error {
|
||||
}
|
||||
conn = resolved
|
||||
}
|
||||
changes, next, perr := s.provider.PollChanges(ctx, src.PluginID, src.CapabilityID, marker, conn, src.SourceConfig)
|
||||
changes, next, perr := s.provider.PollChanges(ctx, src.InstallationID, src.CapabilityID, marker, conn, src.SourceConfig)
|
||||
if perr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: poll changes failed", "source_id", src.ID, "err", perr)
|
||||
if rerr := s.store.RecordError(ctx, src.ID, perr.Error()); rerr != nil {
|
||||
@@ -278,26 +295,22 @@ func (s *Service) PollOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) createEvent(ctx context.Context, src Source, marker string, startedAt time.Time) (int64, bool) {
|
||||
func (s *Service) createEvent(ctx context.Context, src Source, marker string, startedAt time.Time) int64 {
|
||||
if s == nil || s.store == nil {
|
||||
return 0, true
|
||||
return 0
|
||||
}
|
||||
id, err := s.store.CreateEvent(ctx, EventCreate{
|
||||
SourceID: src.ID,
|
||||
PluginID: src.PluginID,
|
||||
CapabilityID: src.CapabilityID,
|
||||
StartedAt: startedAt,
|
||||
MarkerBefore: marker,
|
||||
SourceID: src.ID,
|
||||
InstallationID: src.InstallationID,
|
||||
CapabilityID: src.CapabilityID,
|
||||
StartedAt: startedAt,
|
||||
MarkerBefore: marker,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrPollAlreadyRunning) {
|
||||
slog.DebugContext(ctx, "autoscan: source poll already running", "source_id", src.ID)
|
||||
return 0, false
|
||||
}
|
||||
slog.WarnContext(ctx, "autoscan: create event failed", "source_id", src.ID, "err", err)
|
||||
return 0, true
|
||||
return 0
|
||||
}
|
||||
return id, true
|
||||
return id
|
||||
}
|
||||
|
||||
func (s *Service) finishEvent(ctx context.Context, eventID int64, finish EventFinish) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -13,14 +12,13 @@ import (
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
settings Settings
|
||||
sources []Source
|
||||
connection Connection
|
||||
advanced map[string]string // source ID -> marker
|
||||
recorded map[string]string // source ID -> error message
|
||||
createdEvents []EventCreate
|
||||
events []EventFinish
|
||||
createEventErr error
|
||||
settings Settings
|
||||
sources []Source
|
||||
connection Connection
|
||||
advanced map[string]string // source ID -> marker
|
||||
recorded map[string]string // source ID -> error message
|
||||
createdEvents []EventCreate
|
||||
events []EventFinish
|
||||
}
|
||||
|
||||
func (f *fakeStore) GetSettings(context.Context) (Settings, error) { return f.settings, nil }
|
||||
@@ -53,9 +51,6 @@ func (f *fakeStore) RecordError(_ context.Context, sourceID, msg string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeStore) CreateEvent(_ context.Context, event EventCreate) (int64, error) {
|
||||
if f.createEventErr != nil {
|
||||
return 0, f.createEventErr
|
||||
}
|
||||
f.createdEvents = append(f.createdEvents, event)
|
||||
return int64(len(f.createdEvents)), nil
|
||||
}
|
||||
@@ -70,20 +65,14 @@ type fakeProvider struct {
|
||||
changes map[string][]Change // key: capabilityID, structured changes
|
||||
nextMarker string
|
||||
err error
|
||||
errByCap map[string]error
|
||||
lastConfig map[string]string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeProvider) PollChanges(_ context.Context, _ string, capabilityID, _ string, _ ResolvedConnection, sourceConfig map[string]string) ([]Change, string, error) {
|
||||
f.calls++
|
||||
func (f *fakeProvider) PollChanges(_ context.Context, _ int, capabilityID, _ string, _ ResolvedConnection, sourceConfig map[string]string) ([]Change, string, error) {
|
||||
f.lastConfig = sourceConfig
|
||||
if f.err != nil {
|
||||
return nil, "", f.err
|
||||
}
|
||||
if err := f.errByCap[capabilityID]; err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if changes, ok := f.changes[capabilityID]; ok {
|
||||
return changes, f.nextMarker, nil
|
||||
}
|
||||
@@ -98,7 +87,7 @@ func TestPollOncePassesSourceConfigToProvider(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.cephfs", CapabilityID: "cephfs", Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "cephfs", Enabled: true,
|
||||
SourceConfig: map[string]string{"exclusions": ".downloads\n.recyclebin"},
|
||||
}},
|
||||
}
|
||||
@@ -112,38 +101,6 @@ func TestPollOncePassesSourceConfigToProvider(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollOnceSkipsSourceWhenPollAlreadyRunning(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
createEventErr: ErrPollAlreadyRunning,
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.cephfs", CapabilityID: "cephfs", Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{"cephfs": {"/mnt/media/Movie/movie.mkv"}}, nextMarker: "m1"}
|
||||
q := &recordingQueuer{}
|
||||
svc := newService(store, prov, q, allowSuppressor{})
|
||||
|
||||
if err := svc.PollOnce(context.Background()); err != nil {
|
||||
t.Fatalf("PollOnce: %v", err)
|
||||
}
|
||||
if prov.calls != 0 {
|
||||
t.Fatalf("provider calls = %d, want 0", prov.calls)
|
||||
}
|
||||
if len(q.enqueued) != 0 {
|
||||
t.Fatalf("enqueued scans = %+v, want none", q.enqueued)
|
||||
}
|
||||
if len(store.events) != 0 {
|
||||
t.Fatalf("finished events = %+v, want none", store.events)
|
||||
}
|
||||
if len(store.recorded) != 0 {
|
||||
t.Fatalf("recorded source errors = %+v, want none", store.recorded)
|
||||
}
|
||||
if len(store.advanced) != 0 {
|
||||
t.Fatalf("advanced markers = %+v, want none", store.advanced)
|
||||
}
|
||||
}
|
||||
|
||||
// passthroughConnRes resolves to empty credentials; the engine doesn't inspect
|
||||
// them in tests (the fake provider ignores conn).
|
||||
type passthroughConnRes struct{}
|
||||
@@ -253,7 +210,7 @@ func TestPollOnceEnqueuesDedupedFolders(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{
|
||||
@@ -284,7 +241,7 @@ func TestPollOnceRecordsSuccessfulEvent(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true, Marker: &marker,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true, Marker: &marker,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{
|
||||
@@ -326,7 +283,7 @@ func TestPollOnceRecordsReusedScanCounts(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{
|
||||
@@ -361,7 +318,7 @@ func TestPollOnceAppliesSourceRewritesBeforeEnqueue(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
PathRewrites: []PathRewrite{{From: "/data/tv", To: "/mnt/media/tv"}},
|
||||
}},
|
||||
}
|
||||
@@ -390,7 +347,7 @@ func TestPollOnceStructuredFileChangeEnqueuesExactFile(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.cephfs", CapabilityID: "cephfs", Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "cephfs", Enabled: true,
|
||||
PathRewrites: []PathRewrite{{From: "/ceph/tv", To: "/mnt/media/tv"}},
|
||||
}},
|
||||
}
|
||||
@@ -423,7 +380,7 @@ func TestPollOnceStructuredSubtreeChangeEnqueuesExactSubtree(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.cephfs", CapabilityID: "cephfs", Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "cephfs", Enabled: true,
|
||||
PathRewrites: []PathRewrite{{From: "/ceph/movies", To: "/mnt/media/movies"}},
|
||||
}},
|
||||
}
|
||||
@@ -459,7 +416,7 @@ func TestPollOnceScansDistinctPathsUnderSameFolder(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{
|
||||
@@ -489,7 +446,7 @@ func TestPollOnceStoresOpaqueMarkerVerbatim(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
}},
|
||||
}
|
||||
const opaque = "eyJjdXJzb3IiOiJhYmMxMjMifQ==|2026-06-02T14:10:00Z"
|
||||
@@ -513,7 +470,7 @@ func TestPollOnceHoldsMarkerWhenPathsReturnedButNoneResolve(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{
|
||||
@@ -566,7 +523,7 @@ func TestPollOnceAdvancesMarkerWhenResolvedButAllSuppressed(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
// rewrite /data/tv -> /mnt/media/tv so fakeResolver resolves the paths.
|
||||
PathRewrites: []PathRewrite{{From: "/data/tv", To: "/mnt/media/tv"}},
|
||||
}},
|
||||
@@ -599,7 +556,7 @@ func TestPollOnceAdvancesMarkerWhenZeroPathsReturned(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{"arr": {}}, nextMarker: "m1"}
|
||||
@@ -631,7 +588,7 @@ func TestPollOnceDisabledNoop(t *testing.T) {
|
||||
func TestPollOnceProviderErrorKeepsMarker(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true}},
|
||||
sources: []Source{{ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true}},
|
||||
}
|
||||
prov := &fakeProvider{err: context.DeadlineExceeded}
|
||||
q := &recordingQueuer{}
|
||||
@@ -650,7 +607,7 @@ func TestPollOnceProviderErrorKeepsMarker(t *testing.T) {
|
||||
func TestPollOnceReleasesClaimOnEnqueueFailure(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true}},
|
||||
sources: []Source{{ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{"arr": {"/mnt/media/Show/S01/E01.mkv"}}, nextMarker: "m1"}
|
||||
sup := &recordingSuppressor{}
|
||||
@@ -683,7 +640,7 @@ func TestPollOncePollsConnectionlessSource(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: nil, Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: nil, Enabled: true,
|
||||
}},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{"arr": {"/mnt/media/Show/S01/E01.mkv"}}, nextMarker: "m1"}
|
||||
@@ -709,7 +666,7 @@ func TestPollOnceSkipsSourcePolledTooRecently(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
PollIntervalSeconds: &interval, LastRunAt: &recent,
|
||||
}},
|
||||
}
|
||||
@@ -733,7 +690,7 @@ func TestPollOnceRunsSourcePastItsInterval(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.arr", CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
ID: "s1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1"), Enabled: true,
|
||||
PollIntervalSeconds: &interval, LastRunAt: &old,
|
||||
}},
|
||||
}
|
||||
@@ -751,23 +708,25 @@ func TestPollOnceRunsSourcePastItsInterval(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollOnceRecordsScanSourceResolutionFailure(t *testing.T) {
|
||||
// Two enabled sources: one polls normally, while the other simulates the
|
||||
// production resolver failing because the plugin cannot be resolved.
|
||||
func TestPollOnceSkipsOrphanedSourceQuietly(t *testing.T) {
|
||||
// Two enabled sources: only "arr-live" is still discovered; "arr-gone"'s
|
||||
// plugin has been uninstalled. The orphan must be skipped quietly — no poll,
|
||||
// no RecordError spam — while the live source still polls normally.
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{
|
||||
{ID: "live", PluginID: "silo.autoscan.live", CapabilityID: "arr-live", ConnectionID: strptr("c1"), Enabled: true},
|
||||
{ID: "gone", PluginID: "silo.autoscan.gone", CapabilityID: "arr-gone", ConnectionID: strptr("c1"), Enabled: true},
|
||||
{ID: "live", InstallationID: 1, CapabilityID: "arr-live", ConnectionID: strptr("c1"), Enabled: true},
|
||||
{ID: "gone", InstallationID: 2, CapabilityID: "arr-gone", ConnectionID: strptr("c1"), Enabled: true},
|
||||
},
|
||||
}
|
||||
prov := &fakeProvider{paths: map[string][]string{
|
||||
"arr-live": {"/mnt/media/Show/S01/E01.mkv"},
|
||||
}, errByCap: map[string]error{
|
||||
"arr-gone": errors.New("scan source plugin \"silo.autoscan.gone\" is not installed"),
|
||||
"arr-gone": {"/mnt/media/Other/S01/E01.mkv"},
|
||||
}, nextMarker: "m1"}
|
||||
q := &recordingQueuer{}
|
||||
svc := newService(store, prov, q, allowSuppressor{})
|
||||
// Lister reports only the live capability; the orphan is absent.
|
||||
lister := fakeLister{sources: []DiscoveredSource{{InstallationID: 1, CapabilityID: "arr-live"}}}
|
||||
svc := NewService(store, prov, passthroughConnRes{}, fakeResolver{}, q, allowSuppressor{}, lister)
|
||||
|
||||
if err := svc.PollOnce(context.Background()); err != nil {
|
||||
t.Fatalf("PollOnce: %v", err)
|
||||
@@ -779,15 +738,9 @@ func TestPollOnceRecordsScanSourceResolutionFailure(t *testing.T) {
|
||||
t.Fatalf("expected live source marker advanced")
|
||||
}
|
||||
if _, ok := store.advanced["gone"]; ok {
|
||||
t.Fatalf("failed source must NOT advance")
|
||||
t.Fatalf("orphaned source must NOT poll/advance")
|
||||
}
|
||||
if got := store.recorded["gone"]; !strings.Contains(got, "not installed") {
|
||||
t.Fatalf("failed source error = %q", got)
|
||||
}
|
||||
if len(store.events) != 2 {
|
||||
t.Fatalf("expected one event per source, got %+v", store.events)
|
||||
}
|
||||
if store.events[1].Status != EventStatusError || !strings.Contains(store.events[1].ErrorMessage, "not installed") {
|
||||
t.Fatalf("failed source event = %+v", store.events[1])
|
||||
if _, ok := store.recorded["gone"]; ok {
|
||||
t.Fatalf("orphaned source must be skipped quietly, but an error was recorded: %q", store.recorded["gone"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,11 +45,11 @@ type Change struct {
|
||||
Scope ChangeScope
|
||||
}
|
||||
|
||||
// Source ties a scan_source plugin capability to a connection plus the
|
||||
// Source ties a scan_source plugin capability instance to a connection plus the
|
||||
// host-owned scheduling/bookkeeping state.
|
||||
type Source struct {
|
||||
ID string
|
||||
PluginID string
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
ConnectionID *string // nil until an operator binds a connection
|
||||
Enabled bool
|
||||
@@ -65,7 +65,6 @@ type Source struct {
|
||||
type EventStatus string
|
||||
|
||||
const (
|
||||
EventStatusRunning EventStatus = "running"
|
||||
EventStatusSuccess EventStatus = "success"
|
||||
EventStatusError EventStatus = "error"
|
||||
EventStatusUnresolved EventStatus = "unresolved"
|
||||
@@ -74,7 +73,7 @@ const (
|
||||
type Event struct {
|
||||
ID int64
|
||||
SourceID *string
|
||||
PluginID string
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
StartedAt time.Time
|
||||
CompletedAt time.Time
|
||||
@@ -92,11 +91,11 @@ type Event struct {
|
||||
}
|
||||
|
||||
type EventCreate struct {
|
||||
SourceID string
|
||||
PluginID string
|
||||
CapabilityID string
|
||||
StartedAt time.Time
|
||||
MarkerBefore string
|
||||
SourceID string
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
StartedAt time.Time
|
||||
MarkerBefore string
|
||||
}
|
||||
|
||||
type EventFinish struct {
|
||||
@@ -155,7 +154,7 @@ type ScanWithEvent struct {
|
||||
ScanRunSummary
|
||||
AutoscanEventID *int64
|
||||
SourceID *string
|
||||
PluginID string
|
||||
InstallationID *int
|
||||
CapabilityID string
|
||||
EventStatus EventStatus
|
||||
EventCompletedAt *time.Time
|
||||
|
||||
Vendored
+11
-12
@@ -40,18 +40,17 @@ const (
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
EventScanComplete = "scan_complete"
|
||||
EventMetadataUpdated = "metadata_updated"
|
||||
EventAdminStatsInvalidated = "admin_stats_invalidated"
|
||||
EventPlaybackSessionsChanged = "playback_sessions_changed"
|
||||
EventUserDisabled = "user_disabled"
|
||||
EventUserDeleted = "user_deleted"
|
||||
EventSettingsChanged = "settings_changed"
|
||||
EventMarkerProviderConfigChanged = "marker_provider_config_changed"
|
||||
EventNodePoolChanged = "node_pool_changed"
|
||||
EventOperationalLogAppended = "operational_log_appended"
|
||||
EventAuditLogAppended = "audit_log_appended"
|
||||
EventEventsNotification = "events_notification"
|
||||
EventScanComplete = "scan_complete"
|
||||
EventMetadataUpdated = "metadata_updated"
|
||||
EventAdminStatsInvalidated = "admin_stats_invalidated"
|
||||
EventPlaybackSessionsChanged = "playback_sessions_changed"
|
||||
EventUserDisabled = "user_disabled"
|
||||
EventUserDeleted = "user_deleted"
|
||||
EventSettingsChanged = "settings_changed"
|
||||
EventNodePoolChanged = "node_pool_changed"
|
||||
EventOperationalLogAppended = "operational_log_appended"
|
||||
EventAuditLogAppended = "audit_log_appended"
|
||||
EventEventsNotification = "events_notification"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1089,9 +1089,8 @@ func validateCatalogOverlayQuery(searchQuery string, def QueryDefinition, ruleFi
|
||||
if def.MediaScope != "" &&
|
||||
def.MediaScope != "movie" &&
|
||||
def.MediaScope != "series" &&
|
||||
def.MediaScope != "episode" &&
|
||||
def.MediaScope != "audiobook" {
|
||||
return fmt.Errorf("%w: media_scope must be 'movie', 'series', 'episode', or 'audiobook'", ErrInvalidCatalogRequest)
|
||||
def.MediaScope != "episode" {
|
||||
return fmt.Errorf("%w: media_scope must be 'movie', 'series', or 'episode'", ErrInvalidCatalogRequest)
|
||||
}
|
||||
if def.Match != "" && def.Match != "all" && def.Match != "any" {
|
||||
return fmt.Errorf("%w: match must be 'all' or 'any'", ErrInvalidCatalogRequest)
|
||||
|
||||
@@ -154,51 +154,6 @@ type ItemDetail struct {
|
||||
EffectiveVersionHDR *bool `json:"effective_version_hdr,omitempty"`
|
||||
EffectiveVersionCodecVideo *string `json:"effective_version_codec_video,omitempty"`
|
||||
EffectiveVersionEditionKey *string `json:"effective_version_edition_key,omitempty"`
|
||||
|
||||
// Audiobook-specific detail. Present only when Type == "audiobook".
|
||||
Audiobook *AudiobookDetailExtension `json:"audiobook,omitempty"`
|
||||
}
|
||||
|
||||
type AudiobookDetailExtension struct {
|
||||
Authors []AudiobookPerson `json:"authors"`
|
||||
Narrators []AudiobookPerson `json:"narrators"`
|
||||
Publisher string `json:"publisher,omitempty"`
|
||||
TotalDurationSeconds int `json:"total_duration_seconds"`
|
||||
Series *AudiobookSeriesGroup `json:"series,omitempty"`
|
||||
OtherNarrations []AudiobookNarration `json:"other_narrations"`
|
||||
Related AudiobookRelatedContent `json:"related"`
|
||||
}
|
||||
|
||||
type AudiobookPerson struct {
|
||||
PersonID string `json:"person_id,omitempty"`
|
||||
Name string `json:"name"`
|
||||
PhotoURL string `json:"photo_url,omitempty"`
|
||||
PhotoThumbhash string `json:"photo_thumbhash,omitempty"`
|
||||
}
|
||||
|
||||
type AudiobookRelatedContent struct {
|
||||
AlsoByAuthor []AudiobookRelatedItem `json:"also_by_author"`
|
||||
Similar []AudiobookRelatedItem `json:"similar"`
|
||||
}
|
||||
|
||||
type AudiobookRelatedItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
SeriesIndex *int `json:"series_index,omitempty"`
|
||||
}
|
||||
|
||||
type AudiobookSeriesGroup struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Entries []AudiobookRelatedItem `json:"entries"`
|
||||
}
|
||||
|
||||
type AudiobookNarration struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
Narrators []string `json:"narrators"`
|
||||
}
|
||||
|
||||
// ItemUserState is per-profile viewer state included in item detail responses.
|
||||
@@ -269,7 +224,6 @@ type FileVersion struct {
|
||||
PresentationKind string `json:"presentation_kind,omitempty"`
|
||||
PresentationGroupKey string `json:"presentation_group_key,omitempty"`
|
||||
PresentationPartIndex int `json:"presentation_part_index,omitempty"`
|
||||
PresentationPartTotal int `json:"presentation_part_total,omitempty"`
|
||||
MultiEpisodeStart int `json:"multi_episode_start,omitempty"`
|
||||
MultiEpisodeEnd int `json:"multi_episode_end,omitempty"`
|
||||
EffectiveAudioTrackIndex *int `json:"effective_audio_track_index,omitempty"`
|
||||
@@ -879,9 +833,6 @@ func (s *DetailService) buildMediaItemDetail(ctx context.Context, item *models.M
|
||||
}
|
||||
|
||||
files = FilterMediaFilesByAccess(files, filter)
|
||||
if item.Type == "audiobook" {
|
||||
sortAudiobookMediaFiles(files)
|
||||
}
|
||||
files = s.preparePlaybackFiles(ctx, files)
|
||||
detail.Versions, detail.PlaybackVariants, detail.Subtitles, detail.Intro, detail.Credits, detail.Recap, detail.Preview = s.buildPlaybackInfo(
|
||||
ctx,
|
||||
@@ -892,10 +843,6 @@ func (s *DetailService) buildMediaItemDetail(ctx context.Context, item *models.M
|
||||
detail.OverlaySummary = overlays.BuildSummary(files)
|
||||
}
|
||||
|
||||
if item.Type == "audiobook" {
|
||||
detail.Audiobook = s.buildAudiobookExtension(ctx, item, detail.Versions, crewCredits, filter)
|
||||
}
|
||||
|
||||
// Series folder paths from confirmed claims when available, otherwise from
|
||||
// the file links that currently belong to the item. This keeps provisional
|
||||
// root-scoped series visible for manual match/admin flows without implying
|
||||
@@ -1005,351 +952,6 @@ func splitCastCrew(credits []PersonCredit) ([]CastCredit, []CrewCredit) {
|
||||
return cast, crew
|
||||
}
|
||||
|
||||
func (s *DetailService) buildAudiobookExtension(
|
||||
ctx context.Context,
|
||||
item *models.MediaItem,
|
||||
versions []FileVersion,
|
||||
crew []CrewCredit,
|
||||
filter AccessFilter,
|
||||
) *AudiobookDetailExtension {
|
||||
if item == nil {
|
||||
return nil
|
||||
}
|
||||
totalDuration := 0
|
||||
for _, version := range versions {
|
||||
if version.Duration > 0 {
|
||||
totalDuration += version.Duration
|
||||
}
|
||||
}
|
||||
|
||||
return &AudiobookDetailExtension{
|
||||
Authors: audiobookPeopleFromCrew(crew, models.PersonKindAuthor.String()),
|
||||
Narrators: audiobookPeopleFromCrew(crew, models.PersonKindNarrator.String()),
|
||||
Publisher: firstNonEmptyString(item.Studios),
|
||||
TotalDurationSeconds: totalDuration,
|
||||
Series: s.fetchAudiobookSeries(ctx, item.ContentID, filter),
|
||||
OtherNarrations: s.fetchAudiobookOtherNarrations(ctx, item.ContentID, filter),
|
||||
Related: AudiobookRelatedContent{
|
||||
AlsoByAuthor: s.fetchAudiobookAlsoByAuthor(ctx, item.ContentID, filter),
|
||||
Similar: s.fetchAudiobookSimilarByGenres(ctx, item.ContentID, filter),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func audiobookPeopleFromCrew(crew []CrewCredit, job string) []AudiobookPerson {
|
||||
out := make([]AudiobookPerson, 0)
|
||||
for _, credit := range crew {
|
||||
if credit.Job != job || strings.TrimSpace(credit.Name) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, AudiobookPerson{
|
||||
PersonID: credit.PersonID,
|
||||
Name: credit.Name,
|
||||
PhotoURL: credit.PhotoURL,
|
||||
PhotoThumbhash: credit.PhotoThumbhash,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func firstNonEmptyString(values []string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func appendAudiobookItemAccessConditions(
|
||||
alias string,
|
||||
filter AccessFilter,
|
||||
conditions *[]string,
|
||||
args *[]any,
|
||||
argIdx *int,
|
||||
) bool {
|
||||
if filter.AllowedLibraryIDs != nil {
|
||||
if len(filter.AllowedLibraryIDs) == 0 {
|
||||
return false
|
||||
}
|
||||
*conditions = append(*conditions, fmt.Sprintf(`
|
||||
EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil_allowed
|
||||
WHERE mil_allowed.content_id = %s.content_id
|
||||
AND mil_allowed.media_folder_id = ANY($%d)
|
||||
)`, alias, *argIdx))
|
||||
*args = append(*args, filter.AllowedLibraryIDs)
|
||||
*argIdx = *argIdx + 1
|
||||
}
|
||||
if len(filter.DisabledLibraryIDs) > 0 {
|
||||
if filter.AllowedLibraryIDs == nil {
|
||||
*conditions = append(*conditions, fmt.Sprintf(`
|
||||
EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil_visible
|
||||
WHERE mil_visible.content_id = %s.content_id
|
||||
)`, alias))
|
||||
}
|
||||
*conditions = append(*conditions, fmt.Sprintf(`
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil_disabled
|
||||
WHERE mil_disabled.content_id = %s.content_id
|
||||
AND mil_disabled.media_folder_id = ANY($%d)
|
||||
)`, alias, *argIdx))
|
||||
*args = append(*args, filter.DisabledLibraryIDs)
|
||||
*argIdx = *argIdx + 1
|
||||
}
|
||||
ApplySectionAccessFilter(alias, AccessFilter{MaxContentRating: filter.MaxContentRating}, conditions, args, argIdx)
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *DetailService) fetchAudiobookAlsoByAuthor(ctx context.Context, contentID string, filter AccessFilter) []AudiobookRelatedItem {
|
||||
if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
args := []any{contentID, models.PersonKindAuthor}
|
||||
argIdx := 3
|
||||
conditions := []string{
|
||||
"ip1.content_id = $1",
|
||||
"ip1.kind = $2",
|
||||
"m2.type = 'audiobook'",
|
||||
}
|
||||
if !appendAudiobookItemAccessConditions("m2", filter, &conditions, &args, &argIdx) {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
SELECT m2.content_id, m2.title, COALESCE(m2.year, 0), COALESCE(m2.poster_path, '')
|
||||
FROM item_people ip1
|
||||
JOIN item_people ip2
|
||||
ON ip2.person_id = ip1.person_id
|
||||
AND ip2.kind = ip1.kind
|
||||
AND ip2.content_id <> ip1.content_id
|
||||
JOIN media_items m2 ON m2.content_id = ip2.content_id
|
||||
WHERE %s
|
||||
ORDER BY m2.year DESC NULLS LAST, LOWER(m2.sort_title)
|
||||
LIMIT 12
|
||||
`, strings.Join(conditions, " AND "))
|
||||
rows, err := s.itemRepo.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]AudiobookRelatedItem, 0, 12)
|
||||
seen := make(map[string]struct{})
|
||||
for rows.Next() {
|
||||
var item AudiobookRelatedItem
|
||||
var posterPath string
|
||||
if err := rows.Scan(&item.ContentID, &item.Title, &item.Year, &posterPath); err != nil {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
if _, ok := seen[item.ContentID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[item.ContentID] = struct{}{}
|
||||
item.PosterURL = s.PresignURL(ctx, posterPath, "featured")
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *DetailService) fetchAudiobookSimilarByGenres(ctx context.Context, contentID string, filter AccessFilter) []AudiobookRelatedItem {
|
||||
if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
args := []any{contentID, models.PersonKindAuthor}
|
||||
argIdx := 3
|
||||
conditions := []string{
|
||||
"m.type = 'audiobook'",
|
||||
"m.content_id <> $1",
|
||||
"m.genres && (SELECT array_agg(g) FROM this_genres)",
|
||||
`NOT EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $2
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)`,
|
||||
}
|
||||
if !appendAudiobookItemAccessConditions("m", filter, &conditions, &args, &argIdx) {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
WITH this_genres AS (
|
||||
SELECT unnest(genres) AS g FROM media_items WHERE content_id = $1
|
||||
),
|
||||
this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $1 AND kind = $2
|
||||
)
|
||||
SELECT m.content_id, m.title, COALESCE(m.year, 0), COALESCE(m.poster_path, '')
|
||||
FROM media_items m
|
||||
WHERE %s
|
||||
ORDER BY
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM unnest(m.genres) AS genre(value)
|
||||
WHERE genre.value IN (SELECT g FROM this_genres)
|
||||
) DESC,
|
||||
COALESCE(m.year, 0) DESC,
|
||||
LOWER(m.sort_title)
|
||||
LIMIT 12
|
||||
`, strings.Join(conditions, " AND "))
|
||||
rows, err := s.itemRepo.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]AudiobookRelatedItem, 0, 12)
|
||||
for rows.Next() {
|
||||
var item AudiobookRelatedItem
|
||||
var posterPath string
|
||||
if err := rows.Scan(&item.ContentID, &item.Title, &item.Year, &posterPath); err != nil {
|
||||
return []AudiobookRelatedItem{}
|
||||
}
|
||||
item.PosterURL = s.PresignURL(ctx, posterPath, "featured")
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *DetailService) fetchAudiobookSeries(ctx context.Context, contentID string, filter AccessFilter) *AudiobookSeriesGroup {
|
||||
if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil {
|
||||
return nil
|
||||
}
|
||||
args := []any{contentID}
|
||||
argIdx := 2
|
||||
conditions := []string{
|
||||
"root.content_id = $1",
|
||||
"m.type = 'audiobook'",
|
||||
}
|
||||
if !appendAudiobookItemAccessConditions("m", filter, &conditions, &args, &argIdx) {
|
||||
return nil
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
SELECT
|
||||
m.content_id,
|
||||
m.title,
|
||||
COALESCE(m.year, 0),
|
||||
COALESCE(m.poster_path, ''),
|
||||
s.series_name,
|
||||
s.series_index
|
||||
FROM audiobook_series root
|
||||
JOIN audiobook_series s ON LOWER(s.series_name) = LOWER(root.series_name)
|
||||
JOIN media_items m ON m.content_id = s.content_id
|
||||
WHERE %s
|
||||
ORDER BY s.series_index NULLS LAST, LOWER(m.sort_title)
|
||||
LIMIT 30
|
||||
`, strings.Join(conditions, " AND "))
|
||||
rows, err := s.itemRepo.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var seriesName string
|
||||
entries := make([]AudiobookRelatedItem, 0, 16)
|
||||
for rows.Next() {
|
||||
var (
|
||||
item AudiobookRelatedItem
|
||||
poster string
|
||||
name string
|
||||
index *float64
|
||||
)
|
||||
if err := rows.Scan(&item.ContentID, &item.Title, &item.Year, &poster, &name, &index); err != nil {
|
||||
return nil
|
||||
}
|
||||
if seriesName == "" {
|
||||
seriesName = name
|
||||
}
|
||||
if index != nil {
|
||||
n := int(*index)
|
||||
if float64(n) == *index {
|
||||
item.SeriesIndex = &n
|
||||
}
|
||||
}
|
||||
item.PosterURL = s.PresignURL(ctx, poster, "featured")
|
||||
entries = append(entries, item)
|
||||
}
|
||||
if len(entries) < 2 {
|
||||
return nil
|
||||
}
|
||||
return &AudiobookSeriesGroup{Name: seriesName, Entries: entries}
|
||||
}
|
||||
|
||||
func (s *DetailService) fetchAudiobookOtherNarrations(ctx context.Context, contentID string, filter AccessFilter) []AudiobookNarration {
|
||||
if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil {
|
||||
return []AudiobookNarration{}
|
||||
}
|
||||
const stripRE = `\s*\(?\s*[-:,]?\s*(UK Version:?|US Version:?)?\s*[Rr]ead [Bb]y [^()]+\)?\s*$`
|
||||
args := []any{stripRE, contentID, models.PersonKindAuthor, models.PersonKindNarrator}
|
||||
argIdx := 5
|
||||
conditions := []string{
|
||||
"m.type = 'audiobook'",
|
||||
"m.content_id <> $2",
|
||||
"trim(regexp_replace(m.title, $1, '', 'g')) = (SELECT norm_title FROM this_book)",
|
||||
`EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $3
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)`,
|
||||
}
|
||||
if !appendAudiobookItemAccessConditions("m", filter, &conditions, &args, &argIdx) {
|
||||
return []AudiobookNarration{}
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
WITH this_book AS (
|
||||
SELECT trim(regexp_replace(title, $1, '', 'g')) AS norm_title
|
||||
FROM media_items WHERE content_id = $2
|
||||
),
|
||||
this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $2 AND kind = $3
|
||||
)
|
||||
SELECT
|
||||
m.content_id,
|
||||
m.title,
|
||||
COALESCE(m.year, 0),
|
||||
COALESCE(string_agg(DISTINCT NULLIF(BTRIM(p.name), ''), '|||'), '') AS narrators
|
||||
FROM media_items m
|
||||
LEFT JOIN item_people ipn ON ipn.content_id = m.content_id AND ipn.kind = $4
|
||||
LEFT JOIN people p ON p.id = ipn.person_id
|
||||
WHERE %s
|
||||
GROUP BY m.content_id, m.title, m.year
|
||||
ORDER BY m.year DESC NULLS LAST, m.title
|
||||
LIMIT 8
|
||||
`, strings.Join(conditions, " AND "))
|
||||
rows, err := s.itemRepo.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return []AudiobookNarration{}
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]AudiobookNarration, 0, 4)
|
||||
for rows.Next() {
|
||||
var item AudiobookNarration
|
||||
var narrators string
|
||||
if err := rows.Scan(&item.ContentID, &item.Title, &item.Year, &narrators); err != nil {
|
||||
return []AudiobookNarration{}
|
||||
}
|
||||
item.Narrators = splitDelimitedNames(narrators)
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func splitDelimitedNames(raw string) []string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return []string{}
|
||||
}
|
||||
parts := strings.Split(raw, "|||")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if value := strings.TrimSpace(part); value != "" {
|
||||
out = append(out, value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func seriesFolderPathsFromFiles(files []*models.MediaFile) []string {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
@@ -1965,30 +1567,6 @@ func markerFromRange(start, end *float64) *Marker {
|
||||
return &Marker{Start: *start, End: *end}
|
||||
}
|
||||
|
||||
func sortAudiobookMediaFiles(files []*models.MediaFile) {
|
||||
sort.SliceStable(files, func(i, j int) bool {
|
||||
a, b := files[i], files[j]
|
||||
if a == nil || b == nil {
|
||||
return a != nil
|
||||
}
|
||||
if a.PresentationPartIndex > 0 || b.PresentationPartIndex > 0 {
|
||||
if a.PresentationPartIndex != b.PresentationPartIndex {
|
||||
if a.PresentationPartIndex == 0 {
|
||||
return false
|
||||
}
|
||||
if b.PresentationPartIndex == 0 {
|
||||
return true
|
||||
}
|
||||
return a.PresentationPartIndex < b.PresentationPartIndex
|
||||
}
|
||||
}
|
||||
if a.FilePath != b.FilePath {
|
||||
return strings.Compare(a.FilePath, b.FilePath) < 0
|
||||
}
|
||||
return a.ID < b.ID
|
||||
})
|
||||
}
|
||||
|
||||
func (s *DetailService) buildPlaybackInfo(
|
||||
ctx context.Context,
|
||||
files []*models.MediaFile,
|
||||
@@ -2044,7 +1622,6 @@ func (s *DetailService) buildPlaybackInfo(
|
||||
PresentationKind: f.PresentationKind,
|
||||
PresentationGroupKey: f.PresentationGroupKey,
|
||||
PresentationPartIndex: f.PresentationPartIndex,
|
||||
PresentationPartTotal: f.PresentationPartTotal,
|
||||
MultiEpisodeStart: f.MultiEpisodeStart,
|
||||
MultiEpisodeEnd: f.MultiEpisodeEnd,
|
||||
EffectiveAudioTrackIndex: intPtr(effectiveAudioSelection.Index),
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestSortAudiobookMediaFilesPreservesPresentationPartOrder(t *testing.T) {
|
||||
files := []*models.MediaFile{
|
||||
{ID: 30, FilePath: "/books/book/03.mp3", PresentationPartIndex: 3},
|
||||
{ID: 10, FilePath: "/books/book/01.mp3", PresentationPartIndex: 1},
|
||||
{ID: 20, FilePath: "/books/book/02.mp3", PresentationPartIndex: 2},
|
||||
}
|
||||
|
||||
sortAudiobookMediaFiles(files)
|
||||
|
||||
got := []int{files[0].ID, files[1].ID, files[2].ID}
|
||||
want := []int{10, 20, 30}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("sorted IDs = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortAudiobookMediaFilesFallsBackToPath(t *testing.T) {
|
||||
files := []*models.MediaFile{
|
||||
{ID: 20, FilePath: "/books/book/02.mp3"},
|
||||
{ID: 10, FilePath: "/books/book/01.mp3"},
|
||||
}
|
||||
|
||||
sortAudiobookMediaFiles(files)
|
||||
|
||||
got := []int{files[0].ID, files[1].ID}
|
||||
want := []int{10, 20}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("sorted IDs = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+136
-138
@@ -28,35 +28,65 @@ type RatingFilter struct {
|
||||
Min float64
|
||||
// Limit caps the number of rows returned. Zero or negative means no limit.
|
||||
Limit int
|
||||
// LibraryID, when non-nil, restricts results to items in that library.
|
||||
// LibraryID, when non-nil, restricts results to items in that library via
|
||||
// the media_item_libraries junction table.
|
||||
LibraryID *int
|
||||
// Filter carries viewer-level access constraints (content rating ceiling,
|
||||
// allowed/disabled library sets).
|
||||
Filter AccessFilter
|
||||
}
|
||||
|
||||
// buildRatingThresholdQuery builds the SQL statement and bind args for ListByRatingThreshold.
|
||||
// It returns an empty query string when access rules exclude every
|
||||
// library, signalling the caller to skip the query and return no rows.
|
||||
func buildRatingThresholdQuery(f RatingFilter) (string, []any) {
|
||||
// ListByRatingThreshold returns media items whose rating_imdb is >= f.Min,
|
||||
// ordered by rating_imdb DESC NULLS LAST. Items without an IMDb rating are
|
||||
// always excluded.
|
||||
func (r *DiscoveryRepository) ListByRatingThreshold(ctx context.Context, f RatingFilter) ([]*models.MediaItem, error) {
|
||||
var conditions []string
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
// IMDb rating threshold - NULL ratings are excluded implicitly by >=.
|
||||
// IMDb rating threshold — NULL ratings are excluded implicitly by >=.
|
||||
conditions = append(conditions, fmt.Sprintf("mi.rating_imdb >= $%d", argIdx))
|
||||
args = append(args, f.Min)
|
||||
argIdx++
|
||||
|
||||
if ok := appendDiscoveryLibraryScope(&conditions, &args, &argIdx, f.LibraryID, f.Filter); !ok {
|
||||
return "", nil
|
||||
// Optional single-library pin.
|
||||
fromClause := "media_items mi"
|
||||
if f.LibraryID != nil {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id = $%d", argIdx))
|
||||
args = append(args, *f.LibraryID)
|
||||
argIdx++
|
||||
} else if f.Filter.AllowedLibraryIDs != nil || len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
if f.Filter.AllowedLibraryIDs != nil {
|
||||
if len(f.Filter.AllowedLibraryIDs) == 0 {
|
||||
return []*models.MediaItem{}, nil
|
||||
}
|
||||
placeholders := make([]string, len(f.Filter.AllowedLibraryIDs))
|
||||
for i, id := range f.Filter.AllowedLibraryIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
if len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
placeholders := make([]string, len(f.Filter.DisabledLibraryIDs))
|
||||
for i, id := range f.Filter.DisabledLibraryIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id NOT IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT %s FROM media_items mi WHERE %s ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC",
|
||||
qualifiedItemColumns("mi"),
|
||||
"SELECT %s FROM %s WHERE %s ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC",
|
||||
itemColumns,
|
||||
fromClause,
|
||||
strings.Join(conditions, " AND "),
|
||||
)
|
||||
|
||||
@@ -65,18 +95,6 @@ func buildRatingThresholdQuery(f RatingFilter) (string, []any) {
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
|
||||
return query, args
|
||||
}
|
||||
|
||||
// ListByRatingThreshold returns media items whose rating_imdb is >= f.Min,
|
||||
// ordered by rating_imdb DESC NULLS LAST. Items without an IMDb rating are
|
||||
// always excluded.
|
||||
func (r *DiscoveryRepository) ListByRatingThreshold(ctx context.Context, f RatingFilter) ([]*models.MediaItem, error) {
|
||||
query, args := buildRatingThresholdQuery(f)
|
||||
if query == "" {
|
||||
return []*models.MediaItem{}, nil
|
||||
}
|
||||
|
||||
rows, err := r.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing items by rating threshold: %w", err)
|
||||
@@ -104,50 +122,6 @@ type UnplayedFilter struct {
|
||||
Filter AccessFilter
|
||||
}
|
||||
|
||||
// buildUnplayedHighRatedQuery builds the SQL statement and bind args for ListUnplayedHighRated.
|
||||
// It returns an empty query string when access rules exclude every
|
||||
// library, signalling the caller to skip the query and return no rows.
|
||||
func buildUnplayedHighRatedQuery(f UnplayedFilter) (string, []any) {
|
||||
var conditions []string
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
// IMDb rating threshold.
|
||||
conditions = append(conditions, fmt.Sprintf("mi.rating_imdb >= $%d", argIdx))
|
||||
args = append(args, f.MinRating)
|
||||
argIdx++
|
||||
|
||||
// Items the user has any watch history entry for are excluded.
|
||||
conditions = append(conditions, fmt.Sprintf(`NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_watch_history uwh
|
||||
WHERE uwh.user_id = $%d
|
||||
AND uwh.profile_id = $%d
|
||||
AND uwh.media_item_id = mi.content_id
|
||||
)`, argIdx, argIdx+1))
|
||||
args = append(args, f.UserID, f.ProfileID)
|
||||
argIdx += 2
|
||||
|
||||
if ok := appendDiscoveryLibraryScope(&conditions, &args, &argIdx, nil, f.Filter); !ok {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT %s FROM media_items mi WHERE %s ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC",
|
||||
qualifiedItemColumns("mi"),
|
||||
strings.Join(conditions, " AND "),
|
||||
)
|
||||
|
||||
if f.Limit > 0 {
|
||||
query += fmt.Sprintf(" LIMIT $%d", argIdx)
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
|
||||
return query, args
|
||||
}
|
||||
|
||||
// ListUnplayedHighRated returns high-rated items that the given user/profile has
|
||||
// never started watching. "Never started" means no row exists in
|
||||
// user_watch_history for (user_id, profile_id, media_item_id), regardless of
|
||||
@@ -159,9 +133,67 @@ func (r *DiscoveryRepository) ListUnplayedHighRated(ctx context.Context, f Unpla
|
||||
return nil, fmt.Errorf("ListUnplayedHighRated: UserID and ProfileID are required")
|
||||
}
|
||||
|
||||
query, args := buildUnplayedHighRatedQuery(f)
|
||||
if query == "" {
|
||||
return []*models.MediaItem{}, nil
|
||||
var conditions []string
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
// IMDb rating threshold.
|
||||
conditions = append(conditions, fmt.Sprintf("mi.rating_imdb >= $%d", argIdx))
|
||||
args = append(args, f.MinRating)
|
||||
argIdx++
|
||||
|
||||
// LEFT JOIN exclusion: items the user has any watch history entry for are excluded.
|
||||
// We use a NOT EXISTS subquery rather than an outer join so it composes cleanly
|
||||
// with library access conditions on the main query.
|
||||
conditions = append(conditions, fmt.Sprintf(`NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM user_watch_history uwh
|
||||
WHERE uwh.user_id = $%d
|
||||
AND uwh.profile_id = $%d
|
||||
AND uwh.media_item_id = mi.content_id
|
||||
)`, argIdx, argIdx+1))
|
||||
args = append(args, f.UserID, f.ProfileID)
|
||||
argIdx += 2
|
||||
|
||||
// Library access control.
|
||||
fromClause := "media_items mi"
|
||||
if f.Filter.AllowedLibraryIDs != nil || len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
if f.Filter.AllowedLibraryIDs != nil {
|
||||
if len(f.Filter.AllowedLibraryIDs) == 0 {
|
||||
return []*models.MediaItem{}, nil
|
||||
}
|
||||
placeholders := make([]string, len(f.Filter.AllowedLibraryIDs))
|
||||
for i, id := range f.Filter.AllowedLibraryIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
if len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
placeholders := make([]string, len(f.Filter.DisabledLibraryIDs))
|
||||
for i, id := range f.Filter.DisabledLibraryIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id NOT IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT %s FROM %s WHERE %s ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC",
|
||||
itemColumns,
|
||||
fromClause,
|
||||
strings.Join(conditions, " AND "),
|
||||
)
|
||||
|
||||
if f.Limit > 0 {
|
||||
query += fmt.Sprintf(" LIMIT $%d", argIdx)
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
|
||||
rows, err := r.pool.Query(ctx, query, args...)
|
||||
@@ -193,10 +225,13 @@ type ForgottenFavoritesFilter struct {
|
||||
Filter AccessFilter
|
||||
}
|
||||
|
||||
// buildForgottenFavoritesQuery builds the SQL statement and bind args for ListForgottenFavorites.
|
||||
// It returns an empty query string when access rules exclude every
|
||||
// library, signalling the caller to skip the query and return no rows.
|
||||
func buildForgottenFavoritesQuery(f ForgottenFavoritesFilter) (string, []any) {
|
||||
// ListForgottenFavorites returns high-rated items (rating_imdb >= 7.0) that the
|
||||
// user/profile either has never watched OR last watched more than LookbackDays
|
||||
// ago. Results are ordered by rating_imdb DESC NULLS LAST.
|
||||
func (r *DiscoveryRepository) ListForgottenFavorites(ctx context.Context, f ForgottenFavoritesFilter) ([]*models.MediaItem, error) {
|
||||
if f.UserID <= 0 || strings.TrimSpace(f.ProfileID) == "" {
|
||||
return nil, fmt.Errorf("ListForgottenFavorites: UserID and ProfileID are required")
|
||||
}
|
||||
if f.LookbackDays <= 0 {
|
||||
f.LookbackDays = 365
|
||||
}
|
||||
@@ -220,15 +255,39 @@ func buildForgottenFavoritesQuery(f ForgottenFavoritesFilter) (string, []any) {
|
||||
args = append(args, f.UserID, f.ProfileID, f.LookbackDays)
|
||||
argIdx += 3
|
||||
|
||||
if ok := appendDiscoveryLibraryScope(&conditions, &args, &argIdx, nil, f.Filter); !ok {
|
||||
return "", nil
|
||||
// Library access control.
|
||||
fromClause := "media_items mi"
|
||||
if f.Filter.AllowedLibraryIDs != nil || len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
if f.Filter.AllowedLibraryIDs != nil {
|
||||
if len(f.Filter.AllowedLibraryIDs) == 0 {
|
||||
return []*models.MediaItem{}, nil
|
||||
}
|
||||
placeholders := make([]string, len(f.Filter.AllowedLibraryIDs))
|
||||
for i, id := range f.Filter.AllowedLibraryIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
if len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
placeholders := make([]string, len(f.Filter.DisabledLibraryIDs))
|
||||
for i, id := range f.Filter.DisabledLibraryIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id NOT IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT %s FROM media_items mi WHERE %s ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC",
|
||||
qualifiedItemColumns("mi"),
|
||||
"SELECT %s FROM %s WHERE %s ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC",
|
||||
itemColumns,
|
||||
fromClause,
|
||||
strings.Join(conditions, " AND "),
|
||||
)
|
||||
|
||||
@@ -237,21 +296,6 @@ func buildForgottenFavoritesQuery(f ForgottenFavoritesFilter) (string, []any) {
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
|
||||
return query, args
|
||||
}
|
||||
|
||||
// ListForgottenFavorites returns high-rated items (rating_imdb >= 7.0) that the
|
||||
// user/profile either has never watched OR last watched more than LookbackDays
|
||||
// ago. Results are ordered by rating_imdb DESC NULLS LAST.
|
||||
func (r *DiscoveryRepository) ListForgottenFavorites(ctx context.Context, f ForgottenFavoritesFilter) ([]*models.MediaItem, error) {
|
||||
if f.UserID <= 0 || strings.TrimSpace(f.ProfileID) == "" {
|
||||
return nil, fmt.Errorf("ListForgottenFavorites: UserID and ProfileID are required")
|
||||
}
|
||||
query, args := buildForgottenFavoritesQuery(f)
|
||||
if query == "" {
|
||||
return []*models.MediaItem{}, nil
|
||||
}
|
||||
|
||||
rows, err := r.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing forgotten favorites: %w", err)
|
||||
@@ -264,49 +308,3 @@ func (r *DiscoveryRepository) ListForgottenFavorites(ctx context.Context, f Forg
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func appendDiscoveryLibraryScope(
|
||||
conditions *[]string,
|
||||
args *[]any,
|
||||
argIdx *int,
|
||||
libraryID *int,
|
||||
filter AccessFilter,
|
||||
) bool {
|
||||
if libraryID != nil {
|
||||
whereSQL, scopeArgs, ok := buildLibraryScopeJoin([]int{*libraryID}, nil, *argIdx, "", "mi.content_id")
|
||||
if ok {
|
||||
*conditions = append(*conditions, whereSQL)
|
||||
*args = append(*args, scopeArgs...)
|
||||
*argIdx += len(scopeArgs)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if filter.AllowedLibraryIDs != nil && len(filter.AllowedLibraryIDs) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// buildLibraryScopeJoin uses semi-joins: disabled libraries are an item-level
|
||||
// exclusion, matching the rest of catalog access filtering and avoiding row fanout.
|
||||
if filter.AllowedLibraryIDs == nil && len(filter.DisabledLibraryIDs) > 0 {
|
||||
// In deny-only mode, require at least one library membership. Without this,
|
||||
// stale or provisional media_items rows with no membership pass NOT EXISTS.
|
||||
*conditions = append(*conditions,
|
||||
"EXISTS (SELECT 1 FROM media_item_libraries mil_scope_any WHERE mil_scope_any.content_id = mi.content_id)",
|
||||
)
|
||||
}
|
||||
|
||||
whereSQL, scopeArgs, ok := buildLibraryScopeJoin(
|
||||
filter.AllowedLibraryIDs,
|
||||
filter.DisabledLibraryIDs,
|
||||
*argIdx,
|
||||
"",
|
||||
"mi.content_id",
|
||||
)
|
||||
if ok {
|
||||
*conditions = append(*conditions, whereSQL)
|
||||
*args = append(*args, scopeArgs...)
|
||||
*argIdx += len(scopeArgs)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -4,12 +4,139 @@ package catalog
|
||||
// discovery_repo.go using the same pure-unit approach used throughout this
|
||||
// package: build a query string and assert that key fragments are present,
|
||||
// without needing a live database.
|
||||
//
|
||||
// The helpers themselves execute against a real pool; integration coverage is
|
||||
// provided by whichever recipe resolver exercises them in higher-level tests.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildRatingThresholdQuery is the testable extraction of the SQL generation
|
||||
// logic from ListByRatingThreshold so we can inspect it without a pool.
|
||||
func buildRatingThresholdQuery(f RatingFilter) (string, []any) {
|
||||
var conditions []string
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
conditions = append(conditions, "mi.rating_imdb >= $1")
|
||||
args = append(args, f.Min)
|
||||
argIdx++
|
||||
|
||||
fromClause := "media_items mi"
|
||||
if f.LibraryID != nil {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
conditions = append(conditions, "mil.media_folder_id = $2")
|
||||
args = append(args, *f.LibraryID)
|
||||
argIdx++
|
||||
} else if f.Filter.AllowedLibraryIDs != nil || len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
if f.Filter.AllowedLibraryIDs != nil {
|
||||
if len(f.Filter.AllowedLibraryIDs) == 0 {
|
||||
return "", nil // early empty
|
||||
}
|
||||
placeholders := make([]string, len(f.Filter.AllowedLibraryIDs))
|
||||
for i, id := range f.Filter.AllowedLibraryIDs {
|
||||
placeholders[i] = "$" + itoa(argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, "mil.media_folder_id IN ("+strings.Join(placeholders, ", ")+")")
|
||||
}
|
||||
if len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
placeholders := make([]string, len(f.Filter.DisabledLibraryIDs))
|
||||
for i, id := range f.Filter.DisabledLibraryIDs {
|
||||
placeholders[i] = "$" + itoa(argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, "mil.media_folder_id NOT IN ("+strings.Join(placeholders, ", ")+")")
|
||||
}
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := "SELECT " + itemColumns + " FROM " + fromClause +
|
||||
" WHERE " + strings.Join(conditions, " AND ") +
|
||||
" ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC"
|
||||
|
||||
if f.Limit > 0 {
|
||||
query += " LIMIT $" + itoa(argIdx)
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
return query, args
|
||||
}
|
||||
|
||||
// buildUnplayedHighRatedQuery mirrors the SQL-building portion of
|
||||
// ListUnplayedHighRated for unit testing.
|
||||
func buildUnplayedHighRatedQuery(f UnplayedFilter) (string, []any) {
|
||||
var conditions []string
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
conditions = append(conditions, "mi.rating_imdb >= $1")
|
||||
args = append(args, f.MinRating)
|
||||
argIdx++
|
||||
|
||||
conditions = append(conditions, "NOT EXISTS (\n\t\tSELECT 1\n\t\tFROM user_watch_history uwh\n\t\tWHERE uwh.user_id = $"+itoa(argIdx)+"\n\t\t AND uwh.profile_id = $"+itoa(argIdx+1)+"\n\t\t AND uwh.media_item_id = mi.content_id\n\t)")
|
||||
args = append(args, f.UserID, f.ProfileID)
|
||||
argIdx += 2
|
||||
|
||||
fromClause := "media_items mi"
|
||||
if f.Filter.AllowedLibraryIDs != nil || len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
if f.Filter.AllowedLibraryIDs != nil {
|
||||
if len(f.Filter.AllowedLibraryIDs) == 0 {
|
||||
return "", nil // early empty
|
||||
}
|
||||
placeholders := make([]string, len(f.Filter.AllowedLibraryIDs))
|
||||
for i, id := range f.Filter.AllowedLibraryIDs {
|
||||
placeholders[i] = "$" + itoa(argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, "mil.media_folder_id IN ("+strings.Join(placeholders, ", ")+")")
|
||||
}
|
||||
if len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
placeholders := make([]string, len(f.Filter.DisabledLibraryIDs))
|
||||
for i, id := range f.Filter.DisabledLibraryIDs {
|
||||
placeholders[i] = "$" + itoa(argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, "mil.media_folder_id NOT IN ("+strings.Join(placeholders, ", ")+")")
|
||||
}
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := "SELECT " + itemColumns + " FROM " + fromClause +
|
||||
" WHERE " + strings.Join(conditions, " AND ") +
|
||||
" ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC"
|
||||
|
||||
if f.Limit > 0 {
|
||||
query += " LIMIT $" + itoa(argIdx)
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
return query, args
|
||||
}
|
||||
|
||||
// itoa is a local helper to avoid importing strconv.
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
buf := [20]byte{}
|
||||
pos := len(buf)
|
||||
for n > 0 {
|
||||
pos--
|
||||
buf[pos] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
return string(buf[pos:])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ListByRatingThreshold SQL generation tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -20,7 +147,6 @@ func TestRatingThreshold_BasicQuery(t *testing.T) {
|
||||
Limit: 20,
|
||||
})
|
||||
|
||||
assertQualifiedItemSelect(t, query)
|
||||
if !strings.Contains(query, "mi.rating_imdb >= $1") {
|
||||
t.Fatalf("expected rating threshold predicate, got:\n%s", query)
|
||||
}
|
||||
@@ -55,24 +181,25 @@ func TestRatingThreshold_NoLimitOmitsClause(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRatingThreshold_LibraryIDUsesSemiJoin(t *testing.T) {
|
||||
func TestRatingThreshold_LibraryIDJoin(t *testing.T) {
|
||||
libID := 42
|
||||
query, args := buildRatingThresholdQuery(RatingFilter{
|
||||
Min: 7.0,
|
||||
LibraryID: &libID,
|
||||
})
|
||||
|
||||
assertNoLibraryJoin(t, query)
|
||||
if !strings.Contains(query, "EXISTS (SELECT 1 FROM media_item_libraries mil_scope_in") {
|
||||
t.Fatalf("expected library EXISTS predicate, got:\n%s", query)
|
||||
if !strings.Contains(query, "JOIN media_item_libraries mil") {
|
||||
t.Fatalf("expected library join, got:\n%s", query)
|
||||
}
|
||||
if !strings.Contains(query, "mil_scope_in.media_folder_id = ANY($2)") {
|
||||
t.Fatalf("expected library id predicate at $2, got:\n%s", query)
|
||||
if !strings.Contains(query, "mil.media_folder_id = $2") {
|
||||
t.Fatalf("expected media_folder_id predicate at $2, got:\n%s", query)
|
||||
}
|
||||
if len(args) != 2 {
|
||||
t.Fatalf("expected 2 args (min + library id slice), got %d", len(args))
|
||||
t.Fatalf("expected 2 args (min + library id), got %d", len(args))
|
||||
}
|
||||
if args[1] != 42 {
|
||||
t.Fatalf("expected library id 42 at args[1], got %v", args[1])
|
||||
}
|
||||
assertIntSliceArg(t, args, 1, []int{42})
|
||||
}
|
||||
|
||||
func TestRatingThreshold_AllowedLibraries(t *testing.T) {
|
||||
@@ -83,20 +210,18 @@ func TestRatingThreshold_AllowedLibraries(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
assertNoLibraryJoin(t, query)
|
||||
if !strings.Contains(query, "EXISTS (SELECT 1 FROM media_item_libraries mil_scope_in") {
|
||||
t.Fatalf("expected allowed library EXISTS predicate, got:\n%s", query)
|
||||
if !strings.Contains(query, "JOIN media_item_libraries mil") {
|
||||
t.Fatalf("expected library join, got:\n%s", query)
|
||||
}
|
||||
if !strings.Contains(query, "mil_scope_in.media_folder_id = ANY($2)") {
|
||||
t.Fatalf("expected allowed library ANY clause, got:\n%s", query)
|
||||
if !strings.Contains(query, "mil.media_folder_id IN ($2, $3)") {
|
||||
t.Fatalf("expected allowed library IN clause, got:\n%s", query)
|
||||
}
|
||||
if len(args) != 2 {
|
||||
t.Fatalf("expected 2 args, got %d: %v", len(args), args)
|
||||
if len(args) != 3 {
|
||||
t.Fatalf("expected 3 args, got %d: %v", len(args), args)
|
||||
}
|
||||
assertIntSliceArg(t, args, 1, []int{1, 2})
|
||||
}
|
||||
|
||||
func TestRatingThreshold_DisabledLibrariesUseItemLevelExclusion(t *testing.T) {
|
||||
func TestRatingThreshold_DisabledLibraries(t *testing.T) {
|
||||
query, args := buildRatingThresholdQuery(RatingFilter{
|
||||
Min: 7.0,
|
||||
Filter: AccessFilter{
|
||||
@@ -104,24 +229,19 @@ func TestRatingThreshold_DisabledLibrariesUseItemLevelExclusion(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
assertNoLibraryJoin(t, query)
|
||||
if !strings.Contains(query, "NOT EXISTS (SELECT 1 FROM media_item_libraries mil_scope_out") {
|
||||
t.Fatalf("expected disabled library NOT EXISTS predicate, got:\n%s", query)
|
||||
}
|
||||
if !strings.Contains(query, "mil_scope_out.media_folder_id = ANY($2)") {
|
||||
t.Fatalf("expected disabled library ANY clause, got:\n%s", query)
|
||||
if !strings.Contains(query, "mil.media_folder_id NOT IN ($2)") {
|
||||
t.Fatalf("expected disabled library NOT IN clause, got:\n%s", query)
|
||||
}
|
||||
if len(args) != 2 {
|
||||
t.Fatalf("expected 2 args, got %d: %v", len(args), args)
|
||||
}
|
||||
assertIntSliceArg(t, args, 1, []int{99})
|
||||
}
|
||||
|
||||
func TestRatingThreshold_EmptyAllowedLibrariesReturnsEmptyQuery(t *testing.T) {
|
||||
query, _ := buildRatingThresholdQuery(RatingFilter{
|
||||
Min: 7.0,
|
||||
Filter: AccessFilter{
|
||||
AllowedLibraryIDs: []int{},
|
||||
AllowedLibraryIDs: []int{}, // empty slice → early return
|
||||
},
|
||||
})
|
||||
if query != "" {
|
||||
@@ -141,7 +261,6 @@ func TestUnplayedHighRated_BasicQuery(t *testing.T) {
|
||||
ProfileID: "profile-abc",
|
||||
})
|
||||
|
||||
assertQualifiedItemSelect(t, query)
|
||||
if !strings.Contains(query, "mi.rating_imdb >= $1") {
|
||||
t.Fatalf("expected rating threshold predicate, got:\n%s", query)
|
||||
}
|
||||
@@ -182,14 +301,14 @@ func TestUnplayedHighRated_NoLimitOmitsClause(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnplayedHighRated_NoLibraryScope(t *testing.T) {
|
||||
func TestUnplayedHighRated_NoLibraryJoin(t *testing.T) {
|
||||
query, _ := buildUnplayedHighRatedQuery(UnplayedFilter{
|
||||
MinRating: 7.0,
|
||||
UserID: 1,
|
||||
ProfileID: "p",
|
||||
})
|
||||
if strings.Contains(query, "media_item_libraries") {
|
||||
t.Fatalf("expected no library scope when no library filter is set, got:\n%s", query)
|
||||
if strings.Contains(query, "JOIN media_item_libraries") {
|
||||
t.Fatalf("expected no library join when no library filter is set, got:\n%s", query)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,17 +322,15 @@ func TestUnplayedHighRated_AllowedLibraries(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
assertNoLibraryJoin(t, query)
|
||||
if !strings.Contains(query, "EXISTS (SELECT 1 FROM media_item_libraries mil_scope_in") {
|
||||
t.Fatalf("expected allowed library EXISTS predicate, got:\n%s", query)
|
||||
if !strings.Contains(query, "JOIN media_item_libraries mil") {
|
||||
t.Fatalf("expected library join, got:\n%s", query)
|
||||
}
|
||||
if !strings.Contains(query, "mil_scope_in.media_folder_id = ANY($4)") {
|
||||
t.Fatalf("expected allowed library ANY clause at $4, got:\n%s", query)
|
||||
if !strings.Contains(query, "mil.media_folder_id IN ($4, $5)") {
|
||||
t.Fatalf("expected allowed library IN clause at $4,$5, got:\n%s", query)
|
||||
}
|
||||
if len(args) != 4 {
|
||||
t.Fatalf("expected 4 args, got %d: %v", len(args), args)
|
||||
if len(args) != 5 {
|
||||
t.Fatalf("expected 5 args, got %d: %v", len(args), args)
|
||||
}
|
||||
assertIntSliceArg(t, args, 3, []int{10, 11})
|
||||
}
|
||||
|
||||
func TestUnplayedHighRated_EmptyAllowedLibrariesReturnsEmptyQuery(t *testing.T) {
|
||||
@@ -242,6 +359,8 @@ func TestUnplayedHighRated_ContentRatingFilter(t *testing.T) {
|
||||
t.Fatalf("expected content_rating = ANY filter, got:\n%s", query)
|
||||
}
|
||||
// args: minRating, userID, profileID, then content rating slice (one arg).
|
||||
// Task 5.1 converted the rating ladder from IN ($N, $N+1, ...) to
|
||||
// = ANY($N) with a single bound slice arg.
|
||||
if len(args) != 4 {
|
||||
t.Fatalf("expected exactly 4 args (minRating, userID, profileID, rating slice); got %d: %v", len(args), args)
|
||||
}
|
||||
@@ -251,6 +370,62 @@ func TestUnplayedHighRated_ContentRatingFilter(t *testing.T) {
|
||||
// ListForgottenFavorites SQL generation tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// buildForgottenFavoritesQuery is the testable extraction of the SQL
|
||||
// generation logic from ListForgottenFavorites.
|
||||
func buildForgottenFavoritesQuery(f ForgottenFavoritesFilter) (string, []any) {
|
||||
if f.LookbackDays <= 0 {
|
||||
f.LookbackDays = 365
|
||||
}
|
||||
|
||||
var conditions []string
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
conditions = append(conditions, "mi.rating_imdb >= 7.0")
|
||||
|
||||
conditions = append(conditions, "NOT EXISTS (\n\t\tSELECT 1\n\t\tFROM user_watch_history uwh\n\t\tWHERE uwh.user_id = $"+itoa(argIdx)+"\n\t\t AND uwh.profile_id = $"+itoa(argIdx+1)+"\n\t\t AND uwh.media_item_id = mi.content_id\n\t\t AND uwh.watched_at >= NOW() - ($"+itoa(argIdx+2)+" || ' days')::interval\n\t)")
|
||||
args = append(args, f.UserID, f.ProfileID, f.LookbackDays)
|
||||
argIdx += 3
|
||||
|
||||
fromClause := "media_items mi"
|
||||
if f.Filter.AllowedLibraryIDs != nil || len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
if f.Filter.AllowedLibraryIDs != nil {
|
||||
if len(f.Filter.AllowedLibraryIDs) == 0 {
|
||||
return "", nil // early empty
|
||||
}
|
||||
placeholders := make([]string, len(f.Filter.AllowedLibraryIDs))
|
||||
for i, id := range f.Filter.AllowedLibraryIDs {
|
||||
placeholders[i] = "$" + itoa(argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, "mil.media_folder_id IN ("+strings.Join(placeholders, ", ")+")")
|
||||
}
|
||||
if len(f.Filter.DisabledLibraryIDs) > 0 {
|
||||
placeholders := make([]string, len(f.Filter.DisabledLibraryIDs))
|
||||
for i, id := range f.Filter.DisabledLibraryIDs {
|
||||
placeholders[i] = "$" + itoa(argIdx)
|
||||
args = append(args, id)
|
||||
argIdx++
|
||||
}
|
||||
conditions = append(conditions, "mil.media_folder_id NOT IN ("+strings.Join(placeholders, ", ")+")")
|
||||
}
|
||||
}
|
||||
|
||||
applyAccessFilter("mi", f.Filter, &conditions, &args, &argIdx)
|
||||
|
||||
query := "SELECT " + itemColumns + " FROM " + fromClause +
|
||||
" WHERE " + strings.Join(conditions, " AND ") +
|
||||
" ORDER BY mi.rating_imdb DESC NULLS LAST, mi.content_id ASC"
|
||||
|
||||
if f.Limit > 0 {
|
||||
query += " LIMIT $" + itoa(argIdx)
|
||||
args = append(args, f.Limit)
|
||||
}
|
||||
return query, args
|
||||
}
|
||||
|
||||
func TestForgottenFavorites_BasicQuery(t *testing.T) {
|
||||
query, args := buildForgottenFavoritesQuery(ForgottenFavoritesFilter{
|
||||
LookbackDays: 365,
|
||||
@@ -259,7 +434,6 @@ func TestForgottenFavorites_BasicQuery(t *testing.T) {
|
||||
ProfileID: "profile-xyz",
|
||||
})
|
||||
|
||||
assertQualifiedItemSelect(t, query)
|
||||
if !strings.Contains(query, "mi.rating_imdb >= 7.0") {
|
||||
t.Fatalf("expected 7.0 rating floor, got:\n%s", query)
|
||||
}
|
||||
@@ -303,14 +477,14 @@ func TestForgottenFavorites_NoLimitOmitsClause(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgottenFavorites_NoLibraryScope(t *testing.T) {
|
||||
func TestForgottenFavorites_NoLibraryJoin(t *testing.T) {
|
||||
query, _ := buildForgottenFavoritesQuery(ForgottenFavoritesFilter{
|
||||
LookbackDays: 365,
|
||||
UserID: 1,
|
||||
ProfileID: "p",
|
||||
})
|
||||
if strings.Contains(query, "media_item_libraries") {
|
||||
t.Fatalf("expected no library scope when no library filter is set, got:\n%s", query)
|
||||
if strings.Contains(query, "JOIN media_item_libraries") {
|
||||
t.Fatalf("expected no library join when no library filter is set, got:\n%s", query)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,17 +498,15 @@ func TestForgottenFavorites_AllowedLibraries(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
assertNoLibraryJoin(t, query)
|
||||
if !strings.Contains(query, "EXISTS (SELECT 1 FROM media_item_libraries mil_scope_in") {
|
||||
t.Fatalf("expected allowed library EXISTS predicate, got:\n%s", query)
|
||||
if !strings.Contains(query, "JOIN media_item_libraries mil") {
|
||||
t.Fatalf("expected library join, got:\n%s", query)
|
||||
}
|
||||
if !strings.Contains(query, "mil_scope_in.media_folder_id = ANY($4)") {
|
||||
t.Fatalf("expected allowed library ANY clause at $4, got:\n%s", query)
|
||||
if !strings.Contains(query, "mil.media_folder_id IN ($4, $5)") {
|
||||
t.Fatalf("expected allowed library IN clause at $4,$5, got:\n%s", query)
|
||||
}
|
||||
if len(args) != 4 {
|
||||
t.Fatalf("expected 4 args, got %d: %v", len(args), args)
|
||||
if len(args) != 5 {
|
||||
t.Fatalf("expected 5 args, got %d: %v", len(args), args)
|
||||
}
|
||||
assertIntSliceArg(t, args, 3, []int{10, 11})
|
||||
}
|
||||
|
||||
func TestForgottenFavorites_EmptyAllowedLibrariesReturnsEmptyQuery(t *testing.T) {
|
||||
@@ -353,13 +525,14 @@ func TestForgottenFavorites_EmptyAllowedLibrariesReturnsEmptyQuery(t *testing.T)
|
||||
|
||||
func TestForgottenFavorites_DefaultLookbackApplied(t *testing.T) {
|
||||
query, args := buildForgottenFavoritesQuery(ForgottenFavoritesFilter{
|
||||
LookbackDays: 0,
|
||||
LookbackDays: 0, // should default to 365
|
||||
UserID: 1,
|
||||
ProfileID: "p",
|
||||
})
|
||||
if !strings.Contains(query, "NOT EXISTS") {
|
||||
t.Fatalf("expected NOT EXISTS subquery, got:\n%s", query)
|
||||
}
|
||||
// args[2] should be the defaulted lookback value (365).
|
||||
if len(args) < 3 {
|
||||
t.Fatalf("expected at least 3 args, got %d", len(args))
|
||||
}
|
||||
@@ -367,98 +540,3 @@ func TestForgottenFavorites_DefaultLookbackApplied(t *testing.T) {
|
||||
t.Fatalf("expected lookback default 365, got %v", args[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryQueries_QualifyContentIDAndAvoidLibraryJoin(t *testing.T) {
|
||||
restricted := AccessFilter{AllowedLibraryIDs: []int{1, 2}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
query string
|
||||
}{
|
||||
{"rating_threshold", mustQuery(buildRatingThresholdQuery(RatingFilter{Min: 7, Filter: restricted}))},
|
||||
{"unplayed_high_rated", mustQuery(buildUnplayedHighRatedQuery(UnplayedFilter{MinRating: 7, UserID: 1, ProfileID: "p", Filter: restricted}))},
|
||||
{"forgotten_favorites", mustQuery(buildForgottenFavoritesQuery(ForgottenFavoritesFilter{UserID: 1, ProfileID: "p", LookbackDays: 365, Filter: restricted}))},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assertQualifiedItemSelect(t, tc.query)
|
||||
assertNoLibraryJoin(t, tc.query)
|
||||
if !strings.Contains(tc.query, "EXISTS (SELECT 1 FROM media_item_libraries mil_scope_in") {
|
||||
t.Fatalf("expected allowed library EXISTS predicate, got:\n%s", tc.query)
|
||||
}
|
||||
if strings.Contains(tc.query, "SELECT content_id,") {
|
||||
t.Fatalf("SELECT list uses an unqualified content_id, got:\n%s", tc.query)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryQueries_DisabledLibrariesUseNotExists(t *testing.T) {
|
||||
filter := AccessFilter{DisabledLibraryIDs: []int{9}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
query string
|
||||
}{
|
||||
{"rating_threshold", mustQuery(buildRatingThresholdQuery(RatingFilter{Min: 7, Filter: filter}))},
|
||||
{"unplayed_high_rated", mustQuery(buildUnplayedHighRatedQuery(UnplayedFilter{MinRating: 7, UserID: 1, ProfileID: "p", Filter: filter}))},
|
||||
{"forgotten_favorites", mustQuery(buildForgottenFavoritesQuery(ForgottenFavoritesFilter{UserID: 1, ProfileID: "p", LookbackDays: 365, Filter: filter}))},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assertNoLibraryJoin(t, tc.query)
|
||||
assertDenyOnlyRequiresLibraryMembership(t, tc.query)
|
||||
if !strings.Contains(tc.query, "NOT EXISTS (SELECT 1 FROM media_item_libraries mil_scope_out") {
|
||||
t.Fatalf("expected disabled library NOT EXISTS predicate, got:\n%s", tc.query)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertQualifiedItemSelect(t *testing.T, query string) {
|
||||
t.Helper()
|
||||
if !strings.HasPrefix(query, "SELECT mi.content_id,") {
|
||||
t.Fatalf("SELECT list is not qualified with the mi alias, got:\n%s", query)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoLibraryJoin(t *testing.T, query string) {
|
||||
t.Helper()
|
||||
if strings.Contains(query, "JOIN media_item_libraries") {
|
||||
t.Fatalf("plain library JOIN can fan out discovery rows, got:\n%s", query)
|
||||
}
|
||||
}
|
||||
|
||||
func assertDenyOnlyRequiresLibraryMembership(t *testing.T, query string) {
|
||||
t.Helper()
|
||||
membershipPredicate := "EXISTS (SELECT 1 FROM media_item_libraries mil_scope_any WHERE mil_scope_any.content_id = mi.content_id)"
|
||||
if !strings.Contains(query, membershipPredicate) {
|
||||
t.Fatalf("deny-only library filters must require positive library membership, got:\n%s", query)
|
||||
}
|
||||
if strings.Contains(query, "mil_scope_any.media_folder_id") {
|
||||
t.Fatalf("positive membership predicate should not bind a specific library, got:\n%s", query)
|
||||
}
|
||||
}
|
||||
|
||||
func assertIntSliceArg(t *testing.T, args []any, idx int, want []int) {
|
||||
t.Helper()
|
||||
if idx >= len(args) {
|
||||
t.Fatalf("missing arg %d in %v", idx, args)
|
||||
}
|
||||
got, ok := args[idx].([]int)
|
||||
if !ok {
|
||||
t.Fatalf("arg %d has type %T, want []int", idx, args[idx])
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("arg %d = %v, want %v", idx, got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("arg %d = %v, want %v", idx, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustQuery(query string, _ []any) string { return query }
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
)
|
||||
|
||||
// SettingsStore is the read/write surface over server_settings shared by the
|
||||
// raw *ServerSettingsRepo and the *EncryptedSettingsRepo decorator. Consumers
|
||||
// depend on this interface (rather than the concrete repo) so they
|
||||
// transparently gain at-rest encryption for sensitive keys once the decorator
|
||||
// is wired in.
|
||||
type SettingsStore interface {
|
||||
Get(ctx context.Context, key string) (string, error)
|
||||
Set(ctx context.Context, key, value string) error
|
||||
GetAll(ctx context.Context) (map[string]string, error)
|
||||
}
|
||||
|
||||
// SensitiveSettingKeys is the single source of truth for which server_settings
|
||||
// keys hold secrets. It drives BOTH at-rest encryption (the decorator encrypts
|
||||
// these on write, decrypts them on read) AND admin-API redaction, so the two
|
||||
// can never drift apart.
|
||||
//
|
||||
// It was audited from the config loader's real inputs
|
||||
// (internal/config/db_loader.go), the admin redaction map, and the other known
|
||||
// server_settings consumers — the Audiobookshelf-compat JWT secret
|
||||
// (internal/audiobooks/config.go), the watch-sync OAuth client config
|
||||
// (internal/watchsync/service.go), and the legacy single-instance arr keys —
|
||||
// including the legacy aliases still read as fallbacks (s3.operational_*,
|
||||
// recommendations.openai_api_key) and redis.sentinel_password, which the old
|
||||
// redaction map omitted.
|
||||
//
|
||||
// Adding a key here is only safe when every reader of that key goes through a
|
||||
// cipher-aware path (the decorator, or a repo that calls
|
||||
// secret.Cipher.DecryptIfEncrypted with secret.SettingsAAD). A raw SQL reader
|
||||
// would otherwise see ciphertext after the backfill.
|
||||
var SensitiveSettingKeys = map[string]bool{
|
||||
// Core auth signing secret — no longer the encryption root (SECRET_KEY is),
|
||||
// and now itself encrypted at rest under SECRET_KEY.
|
||||
"auth.jwt_secret": true,
|
||||
// Audiobookshelf-compat HMAC signing key (generated + persisted as hex).
|
||||
"audiobooks.abs.jwt_secret": true,
|
||||
|
||||
// S3 — public assets, private internal, user DB, plus the legacy
|
||||
// "operational" aliases db_loader still reads as fallbacks (L155-192;
|
||||
// migration 086 copied but did not delete them).
|
||||
"s3.public_access_key": true,
|
||||
"s3.public_secret_key": true,
|
||||
"s3.public_token_secret": true,
|
||||
"s3.private_access_key": true,
|
||||
"s3.private_secret_key": true,
|
||||
"s3.user_db_access_key": true,
|
||||
"s3.user_db_secret_key": true,
|
||||
"s3.operational_access_key": true,
|
||||
"s3.operational_secret_key": true,
|
||||
"s3.operational_token_secret": true,
|
||||
|
||||
// Redis — url may embed credentials (redis://:pass@host); sentinel_password
|
||||
// is read at db_loader L334 and was MISSING from the old redaction map.
|
||||
"redis.url": true,
|
||||
"redis.sentinel_password": true,
|
||||
|
||||
// Metadata / list-provider API keys.
|
||||
"tmdb.api_key": true,
|
||||
"mdblist.api_key": true,
|
||||
"introdb.api_key": true,
|
||||
|
||||
// Subtitle AI (on-demand translation) API key.
|
||||
"subtitle_ai.api_key": true,
|
||||
|
||||
// Recommendations embedding auth (+ legacy openai alias at db_loader L409).
|
||||
"recommendations.embedding_auth_token": true,
|
||||
"recommendations.openai_api_key": true,
|
||||
|
||||
// Watch-sync OAuth client credentials. The client_id entries are low-value
|
||||
// but kept as a harmless safe superset and to match existing redaction.
|
||||
"watchsync.trakt.client_id": true,
|
||||
"watchsync.trakt.client_secret": true,
|
||||
"watchsync.simkl.client_id": true,
|
||||
"watchsync.simkl.client_secret": true,
|
||||
|
||||
// Legacy single-instance arr keys (superseded by per-integration rows, but
|
||||
// still referenced by older request_integrations rows until backfilled).
|
||||
"requests.radarr.api_key": true,
|
||||
"requests.sonarr.api_key": true,
|
||||
}
|
||||
|
||||
// EncryptedSettingsRepo decorates a raw settings store, transparently
|
||||
// encrypting the SensitiveSettingKeys on write and decrypting them on read.
|
||||
// config.LoadFromDB and every other consumer keep seeing plaintext while the
|
||||
// values rest as ciphertext in the database, GCM-bound to their key.
|
||||
type EncryptedSettingsRepo struct {
|
||||
// inner is the RAW store (no encryption). It must never be another
|
||||
// EncryptedSettingsRepo, or writes would double-encrypt.
|
||||
inner SettingsStore
|
||||
cipher *secret.Cipher
|
||||
}
|
||||
|
||||
// NewEncryptedSettingsRepo wraps a raw settings store with the cipher. Pass the
|
||||
// concrete *ServerSettingsRepo (or an in-memory fake in tests) as inner.
|
||||
func NewEncryptedSettingsRepo(inner SettingsStore, cipher *secret.Cipher) *EncryptedSettingsRepo {
|
||||
return &EncryptedSettingsRepo{inner: inner, cipher: cipher}
|
||||
}
|
||||
|
||||
var _ SettingsStore = (*EncryptedSettingsRepo)(nil)
|
||||
|
||||
// Set encrypts a sensitive, non-empty value (AAD-bound to its key) before
|
||||
// delegating to the raw store; non-sensitive keys and empty values delegate
|
||||
// unchanged. Encrypting an empty value is a no-op, so an absent secret stays an
|
||||
// empty column rather than an envelope.
|
||||
func (r *EncryptedSettingsRepo) Set(ctx context.Context, key, value string) error {
|
||||
if SensitiveSettingKeys[key] && value != "" {
|
||||
ct, err := r.cipher.Encrypt(value, secret.SettingsAAD(key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt setting %q: %w", key, err)
|
||||
}
|
||||
value = ct
|
||||
}
|
||||
return r.inner.Set(ctx, key, value)
|
||||
}
|
||||
|
||||
// Get reads a value and applies the read-path contract: legacy plaintext passes
|
||||
// through, an enc:v1: value is decrypted, and a corrupt ciphertext errors.
|
||||
func (r *EncryptedSettingsRepo) Get(ctx context.Context, key string) (string, error) {
|
||||
value, err := r.inner.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := r.cipher.DecryptIfEncrypted(value, secret.SettingsAAD(key))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decrypt setting %q: %w", key, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GetAll reads every setting and decrypts any enc:v1: value in place, so
|
||||
// callers (notably config.LoadFromDB) receive a fully plaintext map.
|
||||
func (r *EncryptedSettingsRepo) GetAll(ctx context.Context) (map[string]string, error) {
|
||||
all, err := r.inner.GetAll(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for key, value := range all {
|
||||
out, derr := r.cipher.DecryptIfEncrypted(value, secret.SettingsAAD(key))
|
||||
if derr != nil {
|
||||
return nil, fmt.Errorf("decrypt setting %q: %w", key, derr)
|
||||
}
|
||||
all[key] = out
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// BackfillSensitiveSettings encrypts any plaintext value currently stored under
|
||||
// a sensitive key. It is idempotent (already-encrypted and empty values are
|
||||
// skipped) and best-effort: per-key failures are collected and returned but do
|
||||
// not abort the sweep, so one unreadable row cannot block startup. Returns the
|
||||
// number of keys newly encrypted.
|
||||
func (r *EncryptedSettingsRepo) BackfillSensitiveSettings(ctx context.Context) (int, error) {
|
||||
keys := make([]string, 0, len(SensitiveSettingKeys))
|
||||
for key := range SensitiveSettingKeys {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys) // deterministic order for logs and tests
|
||||
|
||||
encrypted := 0
|
||||
var errs []error
|
||||
for _, key := range keys {
|
||||
// Read the RAW inner value (not through this decorator) so we can tell
|
||||
// plaintext from an already-encrypted envelope.
|
||||
raw, err := r.inner.Get(ctx, key)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("read %q: %w", key, err))
|
||||
continue
|
||||
}
|
||||
if raw == "" || secret.IsEncrypted(raw) {
|
||||
continue
|
||||
}
|
||||
ct, err := r.cipher.Encrypt(raw, secret.SettingsAAD(key))
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("encrypt %q: %w", key, err))
|
||||
continue
|
||||
}
|
||||
if err := r.inner.Set(ctx, key, ct); err != nil {
|
||||
errs = append(errs, fmt.Errorf("write %q: %w", key, err))
|
||||
continue
|
||||
}
|
||||
encrypted++
|
||||
}
|
||||
return encrypted, errors.Join(errs...)
|
||||
}
|
||||
@@ -1,210 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
)
|
||||
|
||||
// memSettings is an in-memory raw SettingsStore for DB-free decorator tests.
|
||||
type memSettings struct{ m map[string]string }
|
||||
|
||||
func newMemSettings() *memSettings { return &memSettings{m: map[string]string{}} }
|
||||
|
||||
func (s *memSettings) Get(_ context.Context, key string) (string, error) { return s.m[key], nil }
|
||||
func (s *memSettings) Set(_ context.Context, key, value string) error {
|
||||
s.m[key] = value
|
||||
return nil
|
||||
}
|
||||
func (s *memSettings) GetAll(_ context.Context) (map[string]string, error) {
|
||||
out := make(map[string]string, len(s.m))
|
||||
for k, v := range s.m {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func newCipher(t *testing.T) *secret.Cipher {
|
||||
t.Helper()
|
||||
key := make([]byte, 48)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
t.Fatalf("rand: %v", err)
|
||||
}
|
||||
c, err := secret.New(key)
|
||||
if err != nil {
|
||||
t.Fatalf("secret.New: %v", err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func pickSensitiveKey() string {
|
||||
for k := range SensitiveSettingKeys {
|
||||
return k
|
||||
}
|
||||
return "auth.jwt_secret"
|
||||
}
|
||||
|
||||
func TestEncryptedSettings_SensitiveRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
raw := newMemSettings()
|
||||
dec := NewEncryptedSettingsRepo(raw, newCipher(t))
|
||||
|
||||
const key = "tmdb.api_key"
|
||||
const plain = "tmdb-secret-key-value"
|
||||
if err := dec.Set(ctx, key, plain); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
// Raw store must hold ciphertext.
|
||||
if got := raw.m[key]; !secret.IsEncrypted(got) {
|
||||
t.Fatalf("raw store value = %q, want enc:v1: ciphertext", got)
|
||||
}
|
||||
// Decorator Get must return plaintext.
|
||||
got, err := dec.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if got != plain {
|
||||
t.Fatalf("Get = %q, want %q", got, plain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSettings_NonSensitivePassthrough(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
raw := newMemSettings()
|
||||
dec := NewEncryptedSettingsRepo(raw, newCipher(t))
|
||||
|
||||
const key = "server.log_level"
|
||||
if SensitiveSettingKeys[key] {
|
||||
t.Fatalf("precondition: %q must be non-sensitive", key)
|
||||
}
|
||||
if err := dec.Set(ctx, key, "debug"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
if got := raw.m[key]; got != "debug" {
|
||||
t.Fatalf("non-sensitive raw value = %q, want plaintext 'debug'", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSettings_LegacyPlaintextPassthrough(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
raw := newMemSettings()
|
||||
dec := NewEncryptedSettingsRepo(raw, newCipher(t))
|
||||
|
||||
// Simulate a pre-encryption row: a sensitive key holding raw plaintext.
|
||||
raw.m["tmdb.api_key"] = "legacy-plaintext"
|
||||
got, err := dec.Get(ctx, "tmdb.api_key")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if got != "legacy-plaintext" {
|
||||
t.Fatalf("legacy plaintext Get = %q, want pass-through", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSettings_EmptySensitiveStaysEmpty(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
raw := newMemSettings()
|
||||
dec := NewEncryptedSettingsRepo(raw, newCipher(t))
|
||||
if err := dec.Set(ctx, "tmdb.api_key", ""); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
if got := raw.m["tmdb.api_key"]; got != "" {
|
||||
t.Fatalf("empty sensitive value stored as %q, want empty (no envelope)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSettings_GetAllDecrypts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
raw := newMemSettings()
|
||||
dec := NewEncryptedSettingsRepo(raw, newCipher(t))
|
||||
|
||||
if err := dec.Set(ctx, "tmdb.api_key", "k1"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
if err := dec.Set(ctx, "server.mode", "integrated"); err != nil {
|
||||
t.Fatalf("Set: %v", err)
|
||||
}
|
||||
all, err := dec.GetAll(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAll: %v", err)
|
||||
}
|
||||
if all["tmdb.api_key"] != "k1" {
|
||||
t.Fatalf("GetAll sensitive = %q, want decrypted 'k1'", all["tmdb.api_key"])
|
||||
}
|
||||
if all["server.mode"] != "integrated" {
|
||||
t.Fatalf("GetAll non-sensitive = %q", all["server.mode"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestSensitiveSettingKeys_Audited locks the audited allowlist: a dropped key is
|
||||
// a plaintext leak (and breaks redaction), so the critical secrets must stay
|
||||
// present, while values that are NOT secrets must stay absent (encrypting them
|
||||
// would corrupt config reads).
|
||||
func TestSensitiveSettingKeys_Audited(t *testing.T) {
|
||||
mustHave := []string{
|
||||
"auth.jwt_secret",
|
||||
"audiobooks.abs.jwt_secret",
|
||||
"s3.public_secret_key",
|
||||
"s3.operational_secret_key", // legacy alias still read as a fallback
|
||||
"redis.sentinel_password", // was missing from the old redaction map
|
||||
"recommendations.openai_api_key",
|
||||
"tmdb.api_key",
|
||||
"requests.radarr.api_key",
|
||||
"requests.sonarr.api_key",
|
||||
"watchsync.trakt.client_secret",
|
||||
}
|
||||
for _, k := range mustHave {
|
||||
if !SensitiveSettingKeys[k] {
|
||||
t.Errorf("SensitiveSettingKeys missing critical secret key %q", k)
|
||||
}
|
||||
}
|
||||
mustNotHave := []string{
|
||||
"server.mode",
|
||||
"server.log_level",
|
||||
"database.max_connections",
|
||||
"jellyfin_compat.server_id",
|
||||
}
|
||||
for _, k := range mustNotHave {
|
||||
if SensitiveSettingKeys[k] {
|
||||
t.Errorf("SensitiveSettingKeys must not contain non-secret key %q", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSettings_BackfillIdempotent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
raw := newMemSettings()
|
||||
dec := NewEncryptedSettingsRepo(raw, newCipher(t))
|
||||
|
||||
key := pickSensitiveKey()
|
||||
raw.m[key] = "plaintext-secret" // legacy row
|
||||
raw.m["server.mode"] = "integrated" // non-sensitive, must be left alone
|
||||
|
||||
n, err := dec.BackfillSensitiveSettings(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("backfill: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("backfill encrypted %d, want 1", n)
|
||||
}
|
||||
if !secret.IsEncrypted(raw.m[key]) {
|
||||
t.Fatalf("after backfill raw[%q] = %q, want ciphertext", key, raw.m[key])
|
||||
}
|
||||
if raw.m["server.mode"] != "integrated" {
|
||||
t.Fatalf("backfill mutated non-sensitive key: %q", raw.m["server.mode"])
|
||||
}
|
||||
// Decryptable back to the original.
|
||||
if got, _ := dec.Get(ctx, key); got != "plaintext-secret" {
|
||||
t.Fatalf("post-backfill Get = %q, want original", got)
|
||||
}
|
||||
// Second run is a no-op.
|
||||
n2, err := dec.BackfillSensitiveSettings(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("backfill 2: %v", err)
|
||||
}
|
||||
if n2 != 0 {
|
||||
t.Fatalf("second backfill encrypted %d, want 0 (no double-encrypt)", n2)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package catalog
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -137,15 +136,3 @@ func TestDeleteInBatchesReturnsError(t *testing.T) {
|
||||
t.Fatalf("expected sentinel, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateOrphanCleanupUsesProvisionalSafetyPredicate(t *testing.T) {
|
||||
conditions := normalizePredicateSQL(orphanedProvisionalMediaItemConditions)
|
||||
for name, query := range map[string]string{
|
||||
"collect": collectOrphanedProvisionalIDsByContentIDSQL,
|
||||
"delete": deleteOrphanedProvisionalIDsByContentIDSQL,
|
||||
} {
|
||||
if !strings.Contains(normalizePredicateSQL(query), conditions) {
|
||||
t.Fatalf("%s late orphan query does not use provisional safety predicate", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+22
-142
@@ -563,6 +563,16 @@ func (r *FolderRepository) DeleteWithStats(
|
||||
}
|
||||
}
|
||||
|
||||
// Filter accumulated image dirs once, now that orphans are gone, against
|
||||
// any surviving content. Empty deleting-set means "exclude nothing".
|
||||
if len(rawDirs) > 0 {
|
||||
filtered, err := filterUnreferencedImageDirs(ctx, r.pool, dirSetToSlice(rawDirs), []string{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats.OrphanedImageDirs = filtered
|
||||
}
|
||||
|
||||
// Phase 2: delete this folder's media_files (folder-tied, not item-tied).
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Deleting media files")
|
||||
@@ -582,29 +592,22 @@ func (r *FolderRepository) DeleteWithStats(
|
||||
}
|
||||
|
||||
// Phase 3: delete remaining folder memberships (shared items kept; only the
|
||||
// membership in this folder is removed). Deleting the memberships with
|
||||
// RETURNING lets us catch skeleton items that a matcher created after the
|
||||
// phase-1 orphan sweep but before the delete job reached this phase.
|
||||
// membership in this folder is removed).
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Removing library memberships")
|
||||
}
|
||||
lateOrphans, lateImageDirs, err := r.deleteFolderMemberships(ctx, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deleting media item links: %w", err)
|
||||
}
|
||||
stats.OrphanedItems += lateOrphans
|
||||
for _, d := range lateImageDirs {
|
||||
rawDirs[d] = struct{}{}
|
||||
}
|
||||
|
||||
// Filter accumulated image dirs once, now that all item orphans are gone,
|
||||
// against any surviving content. Empty deleting-set means "exclude nothing".
|
||||
if len(rawDirs) > 0 {
|
||||
filtered, err := filterUnreferencedImageDirs(ctx, r.pool, dirSetToSlice(rawDirs), []string{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if _, err := deleteInBatches(ctx, folderChildDeleteBatch, func(ctx context.Context) (int64, error) {
|
||||
tag, e := r.pool.Exec(ctx, `
|
||||
DELETE FROM media_item_libraries
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM media_item_libraries WHERE media_folder_id = $1 LIMIT $2
|
||||
)`, id, folderChildDeleteBatch)
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
stats.OrphanedImageDirs = filtered
|
||||
return tag.RowsAffected(), nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting media item links: %w", err)
|
||||
}
|
||||
|
||||
// Phase 4: delete the now-lightweight folder row. Tolerate 0 rows so a
|
||||
@@ -653,129 +656,6 @@ func (r *FolderRepository) collectOrphanBatch(ctx context.Context, folderID, lim
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// deleteFolderMemberships removes this folder's remaining item memberships and
|
||||
// deletes any content that becomes orphaned as a result. This is intentionally
|
||||
// separate from the phase-1 orphan sweep because in-flight matchers can create
|
||||
// new skeleton memberships after phase 1 has already drained.
|
||||
func (r *FolderRepository) deleteFolderMemberships(ctx context.Context, folderID int) (int, []string, error) {
|
||||
var orphaned int
|
||||
var imageDirs []string
|
||||
for {
|
||||
var contentIDs []string
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
ids, err := r.deleteFolderMembershipBatch(ctx, folderID, folderChildDeleteBatch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
contentIDs = ids
|
||||
return nil
|
||||
}); err != nil {
|
||||
return orphaned, imageDirs, err
|
||||
}
|
||||
if len(contentIDs) > 0 {
|
||||
deleted, dirs, err := r.deleteOrphanedItemsByContentID(ctx, contentIDs)
|
||||
if err != nil {
|
||||
return orphaned, imageDirs, err
|
||||
}
|
||||
orphaned += deleted
|
||||
imageDirs = append(imageDirs, dirs...)
|
||||
}
|
||||
if len(contentIDs) < folderChildDeleteBatch {
|
||||
return orphaned, imageDirs, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *FolderRepository) deleteFolderMembershipBatch(ctx context.Context, folderID, limit int) ([]string, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
DELETE FROM media_item_libraries
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM media_item_libraries WHERE media_folder_id = $1 LIMIT $2
|
||||
)
|
||||
RETURNING content_id`, folderID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var contentIDs []string
|
||||
for rows.Next() {
|
||||
var contentID string
|
||||
if err := rows.Scan(&contentID); err != nil {
|
||||
return nil, fmt.Errorf("scanning deleted membership content_id: %w", err)
|
||||
}
|
||||
contentIDs = append(contentIDs, contentID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return contentIDs, nil
|
||||
}
|
||||
|
||||
const collectOrphanedProvisionalIDsByContentIDSQL = `
|
||||
SELECT mi.content_id
|
||||
FROM public.media_items mi
|
||||
WHERE mi.content_id = ANY($1)
|
||||
AND ` + orphanedProvisionalMediaItemConditions
|
||||
|
||||
const deleteOrphanedProvisionalIDsByContentIDSQL = `
|
||||
DELETE FROM public.media_items mi
|
||||
WHERE mi.content_id = ANY($1)
|
||||
AND ` + orphanedProvisionalMediaItemConditions
|
||||
|
||||
func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, contentIDs []string) (int, []string, error) {
|
||||
if len(contentIDs) == 0 {
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
orphanIDs, err := r.collectOrphanedProvisionalIDsByContentID(ctx, contentIDs)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if len(orphanIDs) == 0 {
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
imageDirs, err := collectImageDirs(ctx, r.pool, orphanIDs)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
var deleted int64
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
tag, err := r.pool.Exec(ctx, deleteOrphanedProvisionalIDsByContentIDSQL, orphanIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deleted = tag.RowsAffected()
|
||||
return nil
|
||||
}); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return int(deleted), imageDirs, nil
|
||||
}
|
||||
|
||||
func (r *FolderRepository) collectOrphanedProvisionalIDsByContentID(ctx context.Context, contentIDs []string) ([]string, error) {
|
||||
rows, err := r.pool.Query(ctx, collectOrphanedProvisionalIDsByContentIDSQL, contentIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var orphanIDs []string
|
||||
for rows.Next() {
|
||||
var contentID string
|
||||
if err := rows.Scan(&contentID); err != nil {
|
||||
return nil, fmt.Errorf("scanning orphan content_id: %w", err)
|
||||
}
|
||||
orphanIDs = append(orphanIDs, contentID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return orphanIDs, nil
|
||||
}
|
||||
|
||||
// dirSetToSlice returns the keys of set as a slice (nil if empty).
|
||||
func dirSetToSlice(set map[string]struct{}) []string {
|
||||
if len(set) == 0 {
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
const defaultOrphanedProvisionalCleanupBatchSize = 1000
|
||||
|
||||
const orphanedMediaItemSafetyConditions = `NOT EXISTS (
|
||||
SELECT 1 FROM public.media_item_libraries mil
|
||||
WHERE mil.content_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.media_files mf
|
||||
WHERE mf.content_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.episodes e
|
||||
WHERE e.series_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.seasons s
|
||||
WHERE s.series_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.library_collection_items lci
|
||||
WHERE lci.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.abs_bookmarks ab
|
||||
WHERE ab.library_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.abs_playback_sessions aps
|
||||
WHERE aps.content_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.abs_rss_feeds arf
|
||||
WHERE arf.library_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.downloads d
|
||||
WHERE d.content_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.playback_history_admin pha
|
||||
WHERE pha.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.plex_sync_item_bindings psib
|
||||
WHERE psib.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.plex_sync_item_state psis
|
||||
WHERE psis.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.podcast_feeds pf
|
||||
WHERE pf.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_favorites uf
|
||||
WHERE uf.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_downloads ud
|
||||
WHERE ud.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_history_hidden_items uhhi
|
||||
WHERE uhhi.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_home_item_dismissals uhid
|
||||
WHERE uhid.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_home_item_dismissals uhid_series
|
||||
WHERE uhid_series.series_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_audio_preferences uap
|
||||
WHERE uap.series_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_personal_collection_items upci
|
||||
WHERE upci.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_ratings ur
|
||||
WHERE ur.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_series_playback_preferences uspp
|
||||
WHERE uspp.series_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_subtitle_preferences usp
|
||||
WHERE usp.series_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_watch_history uwh
|
||||
WHERE uwh.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_watch_progress uwp
|
||||
WHERE uwp.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.user_watchlist uwl
|
||||
WHERE uwl.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.watch_provider_favorite_items wpfi
|
||||
WHERE wpfi.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.watch_provider_history_exports wphe
|
||||
WHERE wphe.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.watch_provider_scrobble_sessions wpss
|
||||
WHERE wpss.media_item_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.watch_together_rooms wtr
|
||||
WHERE wtr.selected_content_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.watch_together_suggestions wts
|
||||
WHERE wts.content_id = mi.content_id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public.webhook_sync_item_state wsis
|
||||
WHERE wsis.media_item_id = mi.content_id
|
||||
)`
|
||||
|
||||
const orphanedProvisionalMediaItemConditions = `mi.status IN ('pending', 'unmatched', 'ambiguous')
|
||||
AND ` + orphanedMediaItemSafetyConditions
|
||||
|
||||
const orphanedProvisionalMediaItemPredicate = `
|
||||
WHERE ` + orphanedProvisionalMediaItemConditions
|
||||
|
||||
type OrphanedProvisionalCleanupStats struct {
|
||||
Candidates int
|
||||
Deleted int
|
||||
}
|
||||
|
||||
type OrphanedProvisionalCleaner struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewOrphanedProvisionalCleaner(pool *pgxpool.Pool) *OrphanedProvisionalCleaner {
|
||||
return &OrphanedProvisionalCleaner{pool: pool}
|
||||
}
|
||||
|
||||
func (c *OrphanedProvisionalCleaner) Cleanup(ctx context.Context, batchSize int) (OrphanedProvisionalCleanupStats, error) {
|
||||
var stats OrphanedProvisionalCleanupStats
|
||||
if c == nil || c.pool == nil {
|
||||
return stats, fmt.Errorf("orphaned provisional cleanup is not configured")
|
||||
}
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultOrphanedProvisionalCleanupBatchSize
|
||||
}
|
||||
|
||||
if err := c.pool.QueryRow(ctx,
|
||||
`SELECT COUNT(*) FROM public.media_items mi `+orphanedProvisionalMediaItemPredicate,
|
||||
).Scan(&stats.Candidates); err != nil {
|
||||
return stats, fmt.Errorf("counting orphaned provisional media items: %w", err)
|
||||
}
|
||||
|
||||
for {
|
||||
tag, err := c.pool.Exec(ctx, `
|
||||
WITH candidates AS (
|
||||
SELECT mi.content_id
|
||||
FROM public.media_items mi
|
||||
`+orphanedProvisionalMediaItemPredicate+`
|
||||
ORDER BY mi.content_id ASC
|
||||
LIMIT $1
|
||||
)
|
||||
DELETE FROM public.media_items mi
|
||||
USING candidates c
|
||||
WHERE mi.content_id = c.content_id
|
||||
`, batchSize)
|
||||
if err != nil {
|
||||
return stats, fmt.Errorf("deleting orphaned provisional media items: %w", err)
|
||||
}
|
||||
|
||||
deleted := int(tag.RowsAffected())
|
||||
stats.Deleted += deleted
|
||||
if deleted < batchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func normalizePredicateSQL(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
func TestOrphanedProvisionalPredicatePreservesDurableMediaItemReferences(t *testing.T) {
|
||||
predicate := normalizePredicateSQL(orphanedProvisionalMediaItemPredicate)
|
||||
for _, want := range []string{
|
||||
"public.abs_bookmarks ab WHERE ab.library_item_id = mi.content_id",
|
||||
"public.abs_playback_sessions aps WHERE aps.content_id = mi.content_id",
|
||||
"public.abs_rss_feeds arf WHERE arf.library_item_id = mi.content_id",
|
||||
"public.episodes e WHERE e.series_id = mi.content_id",
|
||||
"public.podcast_feeds pf WHERE pf.media_item_id = mi.content_id",
|
||||
"public.seasons s WHERE s.series_id = mi.content_id",
|
||||
"public.user_audio_preferences uap WHERE uap.series_id = mi.content_id",
|
||||
"public.user_home_item_dismissals uhid WHERE uhid.media_item_id = mi.content_id",
|
||||
"public.user_home_item_dismissals uhid_series WHERE uhid_series.series_id = mi.content_id",
|
||||
"public.user_personal_collection_items upci WHERE upci.media_item_id = mi.content_id",
|
||||
"public.user_series_playback_preferences uspp WHERE uspp.series_id = mi.content_id",
|
||||
"public.user_subtitle_preferences usp WHERE usp.series_id = mi.content_id",
|
||||
} {
|
||||
if !strings.Contains(predicate, normalizePredicateSQL(want)) {
|
||||
t.Fatalf("cleanup predicate missing durable reference guard %q", want)
|
||||
}
|
||||
}
|
||||
for _, droppedTable := range []string{
|
||||
"abs_collection_items",
|
||||
"abs_playlist_items",
|
||||
"abs_playlists",
|
||||
} {
|
||||
if strings.Contains(predicate, droppedTable) {
|
||||
t.Fatalf("cleanup predicate must not reference dropped legacy table %q", droppedTable)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -185,9 +185,8 @@ func (q QueryDefinition) ValidateWithOptions(allowPersonalizedSorts, allowPerson
|
||||
if normalized.MediaScope != "" &&
|
||||
normalized.MediaScope != "movie" &&
|
||||
normalized.MediaScope != "series" &&
|
||||
normalized.MediaScope != "episode" &&
|
||||
normalized.MediaScope != "audiobook" {
|
||||
return fmt.Errorf("media_scope must be 'movie', 'series', 'episode', or 'audiobook'")
|
||||
normalized.MediaScope != "episode" {
|
||||
return fmt.Errorf("media_scope must be 'movie', 'series', or 'episode'")
|
||||
}
|
||||
|
||||
if normalized.Match != "all" && normalized.Match != "any" {
|
||||
|
||||
@@ -60,18 +60,6 @@ func TestValidate_EpisodeMediaScope(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_AudiobookMediaScope(t *testing.T) {
|
||||
qd := QueryDefinition{
|
||||
MediaScope: "audiobook",
|
||||
Match: "all",
|
||||
Groups: []QueryGroup{},
|
||||
Sort: QuerySort{Field: "title", Order: "asc"},
|
||||
}
|
||||
if err := qd.Validate(); err != nil {
|
||||
t.Fatalf("expected audiobook media scope to be valid, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateWithSortScope_RejectsPersonalizedSortWithoutProfileScope(t *testing.T) {
|
||||
qd := QueryDefinition{
|
||||
Match: "all",
|
||||
|
||||
@@ -634,7 +634,7 @@ func (s *Service) exportFolders(ctx context.Context, libraryIDs []int) ([]Librar
|
||||
|
||||
func (s *Service) schemaVersion(ctx context.Context) (int, error) {
|
||||
var version int
|
||||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version_id), 0) FROM public.goose_db_version").Scan(&version); err != nil {
|
||||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_versions").Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("loading schema version: %w", err)
|
||||
}
|
||||
return version, nil
|
||||
|
||||
@@ -32,7 +32,7 @@ func ExtractFrame(ctx context.Context, opts FrameExtractOptions) ([]byte, string
|
||||
runExtract = runFFmpegFrameExtract
|
||||
}
|
||||
|
||||
resolvedAccel := playback.ResolveHWAccelWithFFmpeg(opts.HWAccel, ffmpegPath)
|
||||
resolvedAccel := playback.ResolveHWAccel(opts.HWAccel)
|
||||
resolvedDevice := opts.HWDevice
|
||||
if resolvedDevice == "" && (resolvedAccel == "qsv" || resolvedAccel == "vaapi") {
|
||||
resolvedDevice = playback.PickRenderDevice("")
|
||||
|
||||
@@ -652,7 +652,7 @@ func (s *Service) extractFrameLocal(ctx context.Context, inputPath string, seekS
|
||||
|
||||
func (s *Service) resolveHWConfig() (string, string) {
|
||||
s.hwResolveOnce.Do(func() {
|
||||
s.resolvedHWAccel = playback.ResolveHWAccelWithFFmpeg(s.hwAccel, s.ffmpegPath)
|
||||
s.resolvedHWAccel = playback.ResolveHWAccel(s.hwAccel)
|
||||
s.resolvedHWDevice = s.hwDevice
|
||||
if s.resolvedHWDevice == "" && (s.resolvedHWAccel == "qsv" || s.resolvedHWAccel == "vaapi") {
|
||||
s.resolvedHWDevice = playback.PickRenderDevice("")
|
||||
|
||||
@@ -7,11 +7,6 @@ import (
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
// minSecretKeyLen is the minimum acceptable SECRET_KEY length (in characters).
|
||||
// It mirrors secret.MinMasterKeyLen; the package is not imported here to keep
|
||||
// the bootstrap loader dependency-free, so the two constants must stay in sync.
|
||||
const minSecretKeyLen = 32
|
||||
|
||||
// BootstrapConfig holds the minimal config needed before database connection.
|
||||
type BootstrapConfig struct {
|
||||
DatabaseURL string
|
||||
@@ -19,10 +14,6 @@ type BootstrapConfig struct {
|
||||
Listen string
|
||||
JFListen string
|
||||
Mode string
|
||||
// SecretKey is the master key (raw SECRET_KEY env value) from which the
|
||||
// at-rest credential cipher derives its data key. It lives outside Postgres
|
||||
// so encrypted secrets survive a full database compromise/dump.
|
||||
SecretKey []byte
|
||||
}
|
||||
|
||||
// LoadBootstrap loads bootstrap configuration from a .env file (if it exists)
|
||||
@@ -37,15 +28,6 @@ func LoadBootstrap(envFile string) (*BootstrapConfig, error) {
|
||||
return nil, fmt.Errorf("DATABASE_URL is required (set in .env or environment)")
|
||||
}
|
||||
|
||||
// SECRET_KEY is the at-rest encryption master key. It is required: the server
|
||||
// must never fall back to a zero key or a key derived from another value, or
|
||||
// encrypted credentials would not survive a database dump. godotenv has
|
||||
// already loaded .env above, so dev sets it there.
|
||||
secretKey := os.Getenv("SECRET_KEY")
|
||||
if len(secretKey) < minSecretKeyLen {
|
||||
return nil, fmt.Errorf("SECRET_KEY is required (>=%d chars); generate one with: openssl rand -base64 48", minSecretKeyLen)
|
||||
}
|
||||
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
@@ -69,6 +51,5 @@ func LoadBootstrap(envFile string) (*BootstrapConfig, error) {
|
||||
Listen: ":" + port,
|
||||
JFListen: ":" + jfPort,
|
||||
Mode: mode,
|
||||
SecretKey: []byte(secretKey),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -198,12 +198,13 @@ type authConfigRaw struct {
|
||||
RefreshTokenExpiry string `yaml:"refresh_token_expiry"`
|
||||
}
|
||||
|
||||
// AudiobookshelfCompatConfig holds the dedicated ABS-compat listener setting.
|
||||
// The listener binds its own port (:13378, ABS's
|
||||
// AudiobookshelfCompatConfig holds the dedicated ABS-compat listener
|
||||
// settings. The listener binds its own port (default :13378, ABS's
|
||||
// conventional port) and serves the full ABS protocol — login, libraries,
|
||||
// items, sessions — without colliding with silo's SPA on the main listener.
|
||||
type AudiobookshelfCompatConfig struct {
|
||||
Listen string `yaml:"-"`
|
||||
Listen string `yaml:"listen"`
|
||||
PublicURL string `yaml:"public_url"`
|
||||
}
|
||||
|
||||
// JellyfinCompatConfig holds compatibility proxy settings.
|
||||
|
||||
@@ -356,13 +356,15 @@ func LoadFromDB(m map[string]string) (*Config, error) {
|
||||
cfg.Auth.RefreshTokenExpiry = refreshTokenExpiry
|
||||
|
||||
// AudiobookshelfCompat — dedicated listener for ABS client apps. Keep the
|
||||
// listener off unless ABS compatibility is explicitly enabled.
|
||||
absCompatEnabled, err := boolOr(m, "audiobookshelf_compat.enabled", false)
|
||||
// listener off unless the audiobooks feature flag is explicitly enabled.
|
||||
audiobooksEnabled, err := boolOr(m, "audiobooks.enabled", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if absCompatEnabled {
|
||||
cfg.AudiobookshelfCompat.Listen = ":13378"
|
||||
if audiobooksEnabled {
|
||||
// Default :13378 mirrors the real Audiobookshelf server convention.
|
||||
cfg.AudiobookshelfCompat.Listen = stringOr(m, "audiobookshelf_compat.listen", ":13378")
|
||||
cfg.AudiobookshelfCompat.PublicURL = stringOr(m, "audiobookshelf_compat.public_url", "http://127.0.0.1:13378")
|
||||
}
|
||||
|
||||
// JellyfinCompat
|
||||
|
||||
@@ -34,8 +34,8 @@ func TestLoadFromDBMetadataPresignExpiryRejectsInvalidDuration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromDBAudiobookshelfCompatFlagGatesCompatListener(t *testing.T) {
|
||||
cfg, err := LoadFromDB(map[string]string{"audiobookshelf_compat.enabled": "false"})
|
||||
func TestLoadFromDBAudiobooksFeatureFlagGatesCompatListener(t *testing.T) {
|
||||
cfg, err := LoadFromDB(map[string]string{"audiobooks.enabled": "false"})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromDB() returned error: %v", err)
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func TestLoadFromDBAudiobookshelfCompatFlagGatesCompatListener(t *testing.T) {
|
||||
t.Fatalf("disabled audiobooks listener = %q, want empty", cfg.AudiobookshelfCompat.Listen)
|
||||
}
|
||||
|
||||
cfg, err = LoadFromDB(map[string]string{"audiobookshelf_compat.enabled": "true"})
|
||||
cfg, err = LoadFromDB(map[string]string{"audiobooks.enabled": "true"})
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromDB() returned error: %v", err)
|
||||
}
|
||||
|
||||
+190
-201
@@ -2,244 +2,233 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/pressly/goose/v3/lock"
|
||||
)
|
||||
|
||||
const (
|
||||
schemaMigrationsLockID int64 = 8_034_219_741
|
||||
gooseVersionTable = "public.goose_db_version"
|
||||
)
|
||||
// MigrationFile represents a parsed migration file with its version number,
|
||||
// filename, and SQL content.
|
||||
type MigrationFile struct {
|
||||
Version int
|
||||
Filename string
|
||||
SQL string
|
||||
}
|
||||
|
||||
// RunMigrations applies all pending Goose migrations from fsys/dir.
|
||||
func RunMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) error {
|
||||
provider, err := newMigrationProvider(pool, fsys, dir)
|
||||
const schemaMigrationsLockID int64 = 8_034_219_741
|
||||
|
||||
type migrationDB interface {
|
||||
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error)
|
||||
}
|
||||
|
||||
// RunMigrations applies all pending .up.sql migrations from the given embed.FS.
|
||||
// It creates a schema_versions table to track which migrations have been applied,
|
||||
// parses migration filenames to extract version numbers, and applies each
|
||||
// pending migration within its own transaction.
|
||||
//
|
||||
// The dir parameter specifies the subdirectory within the fs.FS that contains
|
||||
// the migration files.
|
||||
func RunMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) (err error) {
|
||||
conn, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("acquiring migration connection: %w", err)
|
||||
}
|
||||
defer provider.Close()
|
||||
defer conn.Release()
|
||||
|
||||
if _, err := provider.Up(ctx); err != nil {
|
||||
return fmt.Errorf("running goose migrations: %w", err)
|
||||
if err := lockMigrations(ctx, conn); err != nil {
|
||||
return fmt.Errorf("acquiring migration lock: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if unlockErr := unlockMigrations(conn); unlockErr != nil {
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%w; releasing migration lock: %v", err, unlockErr)
|
||||
return
|
||||
}
|
||||
err = fmt.Errorf("releasing migration lock: %w", unlockErr)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := ensureSchemaVersionsTable(ctx, conn); err != nil {
|
||||
return fmt.Errorf("ensuring schema_versions table: %w", err)
|
||||
}
|
||||
|
||||
migrations, err := loadMigrations(fsys, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading migrations: %w", err)
|
||||
}
|
||||
|
||||
applied, err := getAppliedVersions(ctx, conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting applied versions: %w", err)
|
||||
}
|
||||
|
||||
for _, m := range migrations {
|
||||
if applied[m.Version] {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := applyMigration(ctx, conn, m); err != nil {
|
||||
return fmt.Errorf("applying migration %s (version %d): %w", m.Filename, m.Version, err)
|
||||
}
|
||||
applied[m.Version] = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MigrationStatus describes a migration source and whether Goose has applied it.
|
||||
type MigrationStatus struct {
|
||||
Version int64
|
||||
Source string
|
||||
State string
|
||||
AppliedAt time.Time
|
||||
func lockMigrations(ctx context.Context, db migrationDB) error {
|
||||
_, err := db.Exec(ctx, "SELECT pg_advisory_lock($1)", schemaMigrationsLockID)
|
||||
return err
|
||||
}
|
||||
|
||||
// MigrationStatuses returns Goose status using the same legacy bootstrap and locking path as RunMigrations.
|
||||
func MigrationStatuses(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) ([]MigrationStatus, error) {
|
||||
provider, err := newMigrationProvider(pool, fsys, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer provider.Close()
|
||||
func unlockMigrations(db migrationDB) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
gooseStatuses, err := provider.Status(ctx)
|
||||
_, err := db.Exec(ctx, "SELECT pg_advisory_unlock($1)", schemaMigrationsLockID)
|
||||
return err
|
||||
}
|
||||
|
||||
// ensureSchemaVersionsTable creates the schema_versions table if it does not
|
||||
// already exist.
|
||||
func ensureSchemaVersionsTable(ctx context.Context, db migrationDB) error {
|
||||
_, err := db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS public.schema_versions (
|
||||
version INT PRIMARY KEY,
|
||||
filename TEXT NOT NULL,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// loadMigrations reads all .up.sql files from the specified directory within
|
||||
// the filesystem and returns them sorted by version number.
|
||||
func loadMigrations(fsys fs.FS, dir string) ([]MigrationFile, error) {
|
||||
entries, err := fs.ReadDir(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading goose migration status: %w", err)
|
||||
return nil, fmt.Errorf("reading migrations directory %q: %w", dir, err)
|
||||
}
|
||||
|
||||
statuses := make([]MigrationStatus, 0, len(gooseStatuses))
|
||||
for _, status := range gooseStatuses {
|
||||
var version int64
|
||||
var source string
|
||||
if status.Source != nil {
|
||||
version = status.Source.Version
|
||||
source = status.Source.Path
|
||||
var migrations []MigrationFile
|
||||
seenVersions := make(map[int]string)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
statuses = append(statuses, MigrationStatus{
|
||||
Version: version,
|
||||
Source: source,
|
||||
State: string(status.State),
|
||||
AppliedAt: status.AppliedAt,
|
||||
|
||||
name := entry.Name()
|
||||
if !strings.HasSuffix(name, ".up.sql") {
|
||||
continue
|
||||
}
|
||||
|
||||
version, err := parseVersion(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing version from %q: %w", name, err)
|
||||
}
|
||||
if existing, ok := seenVersions[version]; ok {
|
||||
return nil, fmt.Errorf("duplicate migration version %d: %s and %s", version, existing, name)
|
||||
}
|
||||
seenVersions[version] = name
|
||||
|
||||
content, err := fs.ReadFile(fsys, filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading migration file %q: %w", name, err)
|
||||
}
|
||||
|
||||
migrations = append(migrations, MigrationFile{
|
||||
Version: version,
|
||||
Filename: name,
|
||||
SQL: string(content),
|
||||
})
|
||||
}
|
||||
return statuses, nil
|
||||
|
||||
sort.Slice(migrations, func(i, j int) bool {
|
||||
return migrations[i].Version < migrations[j].Version
|
||||
})
|
||||
|
||||
return migrations, nil
|
||||
}
|
||||
|
||||
func newMigrationProvider(pool *pgxpool.Pool, fsys fs.FS, dir string) (*goose.Provider, error) {
|
||||
migrationFS, err := migrationSubFS(fsys, dir)
|
||||
// parseVersion extracts the numeric version prefix from a migration filename.
|
||||
// For example, "001_initial_schema.up.sql" returns 1.
|
||||
func parseVersion(filename string) (int, error) {
|
||||
// Split on the first underscore to get the version prefix.
|
||||
parts := strings.SplitN(filename, "_", 2)
|
||||
if len(parts) < 2 {
|
||||
return 0, fmt.Errorf("migration filename %q must be in format NNN_description.up.sql", filename)
|
||||
}
|
||||
|
||||
version, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid version number in %q: %w", filename, err)
|
||||
}
|
||||
|
||||
if version <= 0 {
|
||||
return 0, fmt.Errorf("version number must be positive, got %d in %q", version, filename)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// getAppliedVersions returns a set of migration versions that have already
|
||||
// been applied.
|
||||
func getAppliedVersions(ctx context.Context, db migrationDB) (map[int]bool, error) {
|
||||
rows, err := db.Query(ctx, "SELECT version FROM public.schema_versions")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sqlDB := stdlib.OpenDBFromPool(pool)
|
||||
|
||||
locker, err := lock.NewPostgresSessionLocker(lock.WithLockID(schemaMigrationsLockID))
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, fmt.Errorf("creating goose migration lock: %w", err)
|
||||
}
|
||||
|
||||
provider, err := goose.NewProvider(
|
||||
goose.DialectPostgres,
|
||||
sqlDB,
|
||||
migrationFS,
|
||||
goose.WithTableName(gooseVersionTable),
|
||||
goose.WithAllowOutofOrder(true),
|
||||
goose.WithSessionLocker(&legacyBootstrapLocker{delegate: locker}),
|
||||
)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, fmt.Errorf("creating goose migration provider: %w", err)
|
||||
}
|
||||
return provider, nil
|
||||
}
|
||||
|
||||
func migrationSubFS(fsys fs.FS, dir string) (fs.FS, error) {
|
||||
if dir == "" || dir == "." {
|
||||
return fsys, nil
|
||||
}
|
||||
|
||||
sub, err := fs.Sub(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening migration directory %q: %w", dir, err)
|
||||
}
|
||||
return sub, nil
|
||||
}
|
||||
|
||||
type legacyBootstrapLocker struct {
|
||||
delegate lock.SessionLocker
|
||||
}
|
||||
|
||||
func (l *legacyBootstrapLocker) SessionLock(ctx context.Context, conn *sql.Conn) error {
|
||||
if err := l.delegate.SessionLock(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bootstrapLegacyGooseVersions(ctx, conn); err != nil {
|
||||
_ = l.delegate.SessionUnlock(context.WithoutCancel(ctx), conn)
|
||||
return fmt.Errorf("bootstrapping goose migration versions: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *legacyBootstrapLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error {
|
||||
return l.delegate.SessionUnlock(ctx, conn)
|
||||
}
|
||||
|
||||
func bootstrapLegacyGooseVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
if err := ensureGooseVersionTable(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureGooseZeroVersion(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasLegacyVersions, err := tableExists(ctx, conn, "public.schema_versions")
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking legacy schema_versions table: %w", err)
|
||||
}
|
||||
if !hasLegacyVersions {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := copyLegacyVersions(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
return verifyLegacyVersions(ctx, conn)
|
||||
}
|
||||
|
||||
func ensureGooseVersionTable(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS public.goose_db_version (
|
||||
id integer PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
|
||||
version_id bigint NOT NULL,
|
||||
is_applied boolean NOT NULL,
|
||||
tstamp timestamp NOT NULL DEFAULT now()
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensuring goose version table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureGooseZeroVersion(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
INSERT INTO public.goose_db_version (version_id, is_applied)
|
||||
SELECT 0, true
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version
|
||||
WHERE version_id = 0 AND is_applied
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensuring goose zero version: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyLegacyVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
INSERT INTO public.goose_db_version (version_id, is_applied, tstamp)
|
||||
SELECT sv.version, true, sv.applied_at::timestamp
|
||||
FROM public.schema_versions sv
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version gv
|
||||
WHERE gv.version_id = sv.version AND gv.is_applied
|
||||
)
|
||||
ORDER BY sv.version
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copying legacy schema_versions rows: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyLegacyVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
rows, err := conn.QueryContext(ctx, `
|
||||
SELECT sv.version
|
||||
FROM public.schema_versions sv
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version gv
|
||||
WHERE gv.version_id = sv.version AND gv.is_applied
|
||||
)
|
||||
ORDER BY sv.version
|
||||
LIMIT 10
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verifying legacy schema_versions rows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var missing []int
|
||||
applied := make(map[int]bool)
|
||||
for rows.Next() {
|
||||
var version int
|
||||
if err := rows.Scan(&version); err != nil {
|
||||
return fmt.Errorf("scanning missing legacy version: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
missing = append(missing, version)
|
||||
applied[version] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("reading missing legacy versions: %w", err)
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return fmt.Errorf("legacy schema_versions rows missing from goose_db_version: %v", missing)
|
||||
}
|
||||
return nil
|
||||
|
||||
return applied, rows.Err()
|
||||
}
|
||||
|
||||
func tableExists(ctx context.Context, conn *sql.Conn, qualifiedName string) (bool, error) {
|
||||
var exists bool
|
||||
if err := conn.QueryRowContext(ctx, "SELECT to_regclass($1) IS NOT NULL", qualifiedName).Scan(&exists); err != nil {
|
||||
return false, err
|
||||
// applyMigration runs a single migration within a transaction and records the
|
||||
// version in schema_versions.
|
||||
func applyMigration(ctx context.Context, db migrationDB, m MigrationFile) error {
|
||||
tx, err := db.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning transaction: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
defer func() {
|
||||
// Rollback is a no-op if the tx has already been committed.
|
||||
_ = tx.Rollback(ctx)
|
||||
}()
|
||||
|
||||
if _, err := tx.Exec(ctx, m.SQL); err != nil {
|
||||
return fmt.Errorf("executing SQL: %w", err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
"INSERT INTO public.schema_versions (version, filename) VALUES ($1, $2)",
|
||||
m.Version, m.Filename,
|
||||
); err != nil {
|
||||
return fmt.Errorf("recording version: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("committing transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user