Compare commits
124
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc3d7824f9 | ||
|
|
8041ba9972 | ||
|
|
14b7877be7 | ||
|
|
58859dec3b | ||
|
|
6e5d096fdb | ||
|
|
9f73ac6f1a | ||
|
|
177cfdc485 | ||
|
|
efd377fbdc | ||
|
|
de89f0136c | ||
|
|
08a0704a11 | ||
|
|
cd946b30a3 | ||
|
|
dc0392dee0 | ||
|
|
681c846da8 | ||
|
|
54a04958b7 | ||
|
|
621349a085 | ||
|
|
ea3b5a2e29 | ||
|
|
a6b510c7c5 | ||
|
|
0fb06670d9 | ||
|
|
8bed6c6452 | ||
|
|
bc921cac0e | ||
|
|
9b82535190 | ||
|
|
cb1bc6ab62 | ||
|
|
07e50b72b6 | ||
|
|
af2c54c8f0 | ||
|
|
3f55440752 | ||
|
|
0a27dac9cd | ||
|
|
9e601d4d56 | ||
|
|
14b54cab0c | ||
|
|
f11bd7ba05 | ||
|
|
dd99e3e54c | ||
|
|
0b5f815c7b | ||
|
|
4f2e94c8c5 | ||
|
|
669050d025 | ||
|
|
13f8d7bc23 | ||
|
|
e441d2d6e9 | ||
|
|
179506e8be | ||
|
|
123d05b7d3 | ||
|
|
c76ec2b61a | ||
|
|
20da980cc8 | ||
|
|
a2d8118978 | ||
|
|
0cdd6c543f | ||
|
|
9123cfb1f4 | ||
|
|
1bfbdd2826 | ||
|
|
449fcf394c | ||
|
|
9c56f40170 | ||
|
|
96256ecbf8 | ||
|
|
bdc46b8c6c | ||
|
|
66cfb0750d | ||
|
|
de8b8189d7 | ||
|
|
4c3911b5e1 | ||
|
|
c2d8dbd746 | ||
|
|
c32b5703ac | ||
|
|
431341c235 | ||
|
|
4dc0522cf2 | ||
|
|
2bd0edb264 | ||
|
|
1ba76af353 | ||
|
|
62b3861da0 | ||
|
|
f113b32de0 | ||
|
|
58fc4abca8 | ||
|
|
3fe38c8c63 | ||
|
|
32302e7b17 | ||
|
|
98d2596fbf | ||
|
|
943b5dd9f0 | ||
|
|
ffef55283a | ||
|
|
81d1644cd7 | ||
|
|
59874c717d | ||
|
|
d27f21b10f | ||
|
|
13081f7d07 | ||
|
|
0025fae492 | ||
|
|
b1c36ad04e | ||
|
|
b2b5c2ade2 | ||
|
|
9c713d43bf | ||
|
|
80120aad02 | ||
|
|
6848fe32fd | ||
|
|
feed82c15d | ||
|
|
ecded4c94f | ||
|
|
6faec2b2a9 | ||
|
|
7e4cacd151 | ||
|
|
1624c77fbd | ||
|
|
1d5b1f4f81 | ||
|
|
6c8ddac627 | ||
|
|
f4d6966404 | ||
|
|
7161c86f73 | ||
|
|
8b82906c7b | ||
|
|
b7979e2f9f | ||
|
|
f8493a47f1 | ||
|
|
5c3c92da28 | ||
|
|
731cdd35fc | ||
|
|
dfa0f6861f | ||
|
|
196f753d1a | ||
|
|
c9294947b1 | ||
|
|
731235f6a5 | ||
|
|
486ecafbb5 | ||
|
|
0be713b30c | ||
|
|
485c6e28fd | ||
|
|
e159f0b4ba | ||
|
|
1ab4e1f269 | ||
|
|
9272203d8c | ||
|
|
0e73e6c240 | ||
|
|
b403954ced | ||
|
|
3f49271f63 | ||
|
|
5d6be3ad4b | ||
|
|
416fba8dab | ||
|
|
17d156a7ca | ||
|
|
6c071e7248 | ||
|
|
c89d31d0db | ||
|
|
b5e4213e81 | ||
|
|
24f516077c | ||
|
|
747eef2c6b | ||
|
|
f3cd1e6ab5 | ||
|
|
e1320c8817 | ||
|
|
7922224839 | ||
|
|
534f7bb901 | ||
|
|
f2b6eff29d | ||
|
|
5fbe4d24f6 | ||
|
|
773148486e | ||
|
|
74e9be6443 | ||
|
|
c3c806410f | ||
|
|
d9b57f28e2 | ||
|
|
49efc3846c | ||
|
|
0bb1030ca2 | ||
|
|
590f61a550 | ||
|
|
5bda20740c | ||
|
|
2637061601 |
@@ -20,6 +20,19 @@ SILO_IMAGE=ghcr.io/silo-server/silo-server:latest
|
||||
POSTGRES_USER=silo
|
||||
POSTGRES_PASSWORD=silo
|
||||
POSTGRES_DB=silo
|
||||
POSTGRES_SHM_SIZE=8gb
|
||||
|
||||
# PostgreSQL auto-tuning is enabled by default in docker-compose.yml. Silo
|
||||
# applies pgtune-style OLTP recommendations through ALTER SYSTEM at startup.
|
||||
# Set POSTGRES_TUNE=off if you manage PostgreSQL tuning yourself.
|
||||
# POSTGRES_TUNE=off
|
||||
# POSTGRES_TUNE_PROFILE=oltp
|
||||
# POSTGRES_TUNE_MEMORY=auto
|
||||
# POSTGRES_TUNE_MEMORY_BUDGET_PERCENT=75
|
||||
# POSTGRES_TUNE_CPUS=auto
|
||||
# POSTGRES_TUNE_STORAGE=ssd
|
||||
# POSTGRES_TUNE_DB_SIZE=auto
|
||||
# POSTGRES_TUNE_CONNECTIONS=100
|
||||
|
||||
# Optional host port overrides for docker-compose.yml.
|
||||
# PORT=8090
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ RUN apt-get update && \
|
||||
echo "deb [signed-by=/usr/share/keyrings/jellyfin.gpg arch=${TARGETARCH}] https://repo.jellyfin.org/debian bookworm main" \
|
||||
> /etc/apt/sources.list.d/jellyfin.list && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 fonts-noto-core fonts-noto-cjk && \
|
||||
apt-get purge -y gnupg && apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN mkdir -p /tmp/silo-transcode
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ RUN apt-get update && \
|
||||
echo "deb [signed-by=/usr/share/keyrings/jellyfin.gpg arch=${TARGETARCH}] https://repo.jellyfin.org/debian bookworm main" \
|
||||
> /etc/apt/sources.list.d/jellyfin.list && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 fonts-noto-core fonts-noto-cjk && \
|
||||
apt-get purge -y gnupg && apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN mkdir -p /tmp/silo-transcode
|
||||
|
||||
@@ -14,13 +14,20 @@ endif
|
||||
|
||||
JELLYFIN_WEB_OUTPUT_DIR ?= third_party/jellyfin-web
|
||||
|
||||
# Build version stamping: inject the git revision so the admin Build panel shows a
|
||||
# version even when Go's VCS metadata isn't embedded (mirrors the Dockerfile ldflags).
|
||||
BUILDINFO_PKG := github.com/Silo-Server/silo-server/internal/buildinfo
|
||||
BUILD_REVISION ?= $(shell git rev-parse HEAD 2>/dev/null)
|
||||
BUILD_DIRTY ?= $(shell test -n "$$(git status --porcelain 2>/dev/null)" && echo true || echo false)
|
||||
GO_LDFLAGS := -X $(BUILDINFO_PKG).revisionOverride=$(BUILD_REVISION) -X $(BUILDINFO_PKG).dirtyOverride=$(BUILD_DIRTY)
|
||||
|
||||
# Build the frontend (requires pnpm)
|
||||
frontend:
|
||||
cd web && pnpm install --frozen-lockfile && pnpm run build
|
||||
|
||||
# Build the Go binary (depends on frontend)
|
||||
build: frontend
|
||||
go build -o silo ./cmd/silo/
|
||||
go build -ldflags "$(GO_LDFLAGS)" -o silo ./cmd/silo/
|
||||
|
||||
# Run frontend dev server (proxies API to localhost:8080)
|
||||
dev-frontend:
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
A self-hosted media streaming server with a React frontend and Go backend. Supports direct play, remuxing, and hardware-accelerated transcoding. Compatible with Jellyfin/Emby clients (VidHub, Findroid) via a built-in compatibility layer.
|
||||
|
||||
Join the community on [Discord](https://discord.gg/4RxuUQAEnW).
|
||||
|
||||
## Deploy with Docker (recommended)
|
||||
|
||||
The easiest way to run Silo is with Docker Compose. The default stack assumes you do not already have PostgreSQL and Redis available, so it bundles PostgreSQL, Redis, FFmpeg, and the application for a one-command start.
|
||||
@@ -216,93 +218,57 @@ Silo requires only a `DATABASE_URL` when running from source or against external
|
||||
| `proxy` | Stream proxy node that connects to the shared deployment database and Redis |
|
||||
| `transcode` | HLS transcode worker node that connects to the shared deployment database and Redis |
|
||||
|
||||
## PostgreSQL Performance Tuning
|
||||
## PostgreSQL Auto-Tuning
|
||||
|
||||
The default PostgreSQL configuration is conservative and not optimized for any particular workload. For better performance, use [PGTune](https://pgtune.leopard.in.ua/) to generate settings tailored to your hardware.
|
||||
|
||||
**Recommended PGTune settings:**
|
||||
|
||||
| Setting | Value |
|
||||
|---|---|
|
||||
| DB Type | **Web application** |
|
||||
| Total Memory | Your server's RAM |
|
||||
| Number of CPUs | Your server's CPU count |
|
||||
| Number of Connections | 20 (Silo's default pool size) |
|
||||
| Data Storage | SSD or HDD depending on your setup |
|
||||
|
||||
Silo's workload is predominantly read-heavy API queries (browse, search, metadata lookups) with the database small enough to fit in RAM for most libraries. The **Web application** profile optimizes for this: CPU-bound simple queries served from cached pages, high concurrent reads, and low write volume.
|
||||
|
||||
If you have a very large library (50,000+ items) or heavy activity logging, consider switching to the **OLTP** profile instead.
|
||||
|
||||
### Recommended Defaults for Modern Hardware
|
||||
|
||||
Assuming you're using modern hardware with fast NVMe storage this is what I'm currently running.
|
||||
|
||||
```ini
|
||||
# Connections
|
||||
max_connections = 100
|
||||
|
||||
# Memory (tuned to keep PG under ~8GB total)
|
||||
shared_buffers = 4GB # holds the entire DB in cache (typical DB is 1-4GB)
|
||||
effective_cache_size = 6GB # planner hint only, not an allocation
|
||||
work_mem = 32MB # per-operation sort/hash memory
|
||||
maintenance_work_mem = 512MB # VACUUM, CREATE INDEX, catalog imports
|
||||
|
||||
# WAL / Write Performance
|
||||
wal_buffers = 64MB
|
||||
min_wal_size = 512MB
|
||||
max_wal_size = 2GB
|
||||
checkpoint_completion_target = 0.9
|
||||
|
||||
# Storage (NVMe/Optane)
|
||||
random_page_cost = 1.1 # default 4.0; NVMe random ≈ sequential
|
||||
effective_io_concurrency = 200 # NVMe handles many concurrent requests
|
||||
|
||||
# Parallelism
|
||||
max_worker_processes = 8
|
||||
max_parallel_workers_per_gather = 4
|
||||
max_parallel_workers = 8
|
||||
max_parallel_maintenance_workers = 4
|
||||
|
||||
# Huge Pages
|
||||
huge_pages = try
|
||||
```
|
||||
|
||||
These settings keep PostgreSQL under ~8GB of RAM while still caching the entire database. For HDD storage, change `random_page_cost` to `4.0` and `effective_io_concurrency` to `2`.
|
||||
|
||||
### Applying Settings
|
||||
|
||||
**Docker (recommended):** Save the settings above to a `postgresql.conf` file and mount it into your container:
|
||||
The default Docker Compose stack does not require a checked-in `postgresql.conf`.
|
||||
It enables Silo's [pgtune](https://github.com/le0pard/pgtune)-style OLTP tuning
|
||||
by default:
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
volumes:
|
||||
- ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
||||
POSTGRES_TUNE: auto
|
||||
```
|
||||
|
||||
Then restart: `docker compose restart postgres`
|
||||
When enabled, Silo connects with `DATABASE_URL` and applies recommendations with
|
||||
`ALTER SYSTEM`, which writes to PostgreSQL's `postgresql.auto.conf` inside the
|
||||
database data directory. Reloadable settings are applied immediately with
|
||||
`pg_reload_conf()`. Settings that PostgreSQL marks as restart-only are written
|
||||
too, and Silo logs the setting names so you can restart PostgreSQL once:
|
||||
|
||||
**Existing PostgreSQL:** You can apply settings without editing config files using `ALTER SYSTEM`, which writes to `postgresql.auto.conf` inside the data directory:
|
||||
|
||||
```sql
|
||||
ALTER SYSTEM SET work_mem = '32MB';
|
||||
ALTER SYSTEM SET effective_cache_size = '6GB';
|
||||
ALTER SYSTEM SET random_page_cost = 1.1;
|
||||
ALTER SYSTEM SET effective_io_concurrency = 200;
|
||||
-- Apply without restart:
|
||||
SELECT pg_reload_conf();
|
||||
|
||||
-- These require a full restart:
|
||||
ALTER SYSTEM SET shared_buffers = '4GB';
|
||||
ALTER SYSTEM SET max_connections = 100;
|
||||
ALTER SYSTEM SET max_worker_processes = 8;
|
||||
ALTER SYSTEM SET huge_pages = 'try';
|
||||
```sh
|
||||
docker compose restart postgres
|
||||
```
|
||||
|
||||
Settings like `shared_buffers`, `max_connections`, `max_worker_processes`, and `huge_pages` require a PostgreSQL restart. Memory/planner settings (`work_mem`, `effective_cache_size`, `random_page_cost`) can be reloaded live.
|
||||
The default Compose database user has the required PostgreSQL permissions. If
|
||||
you use an external PostgreSQL server, make sure the configured `DATABASE_URL`
|
||||
user can run `ALTER SYSTEM`, or set `POSTGRES_TUNE=off` and manage
|
||||
PostgreSQL yourself.
|
||||
|
||||
For `POSTGRES_TUNE_MEMORY=auto`, Silo uses the first trustworthy memory source:
|
||||
a finite Docker cgroup limit, the read-only `/host/proc/meminfo` mount supplied
|
||||
by the bundled Compose file, then `/proc/meminfo` with container safety guards.
|
||||
Auto-detected memory is treated as a PostgreSQL budget, defaulting to 75% of
|
||||
detected RAM so Silo, Redis, plugins, transcodes, and the OS retain headroom.
|
||||
`POSTGRES_TUNE_DB_SIZE=auto` queries `pg_database_size(current_database())` and
|
||||
classifies the workload by comparing the database size to that memory budget.
|
||||
|
||||
Optional tuning overrides:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---:|---|
|
||||
| `POSTGRES_TUNE_PROFILE` | `oltp` | Tuning profile. Only `oltp` is currently supported. |
|
||||
| `POSTGRES_TUNE_MEMORY` | `auto` | Server/container RAM, such as `8GB` or `32GB`; explicit values are used as-is. |
|
||||
| `POSTGRES_TUNE_MEMORY_BUDGET_PERCENT` | `75` | Percent of auto-detected RAM used for PostgreSQL recommendations. |
|
||||
| `POSTGRES_TUNE_CPUS` | `auto` | CPU count used for worker recommendations. |
|
||||
| `POSTGRES_TUNE_STORAGE` | `ssd` | One of `hdd`, `ssd`, `san`, or `nvme`. |
|
||||
| `POSTGRES_TUNE_DB_SIZE` | `auto` | Use `less_ram` when the database comfortably fits in RAM, `mid_ram`, or `greater_ram` for very large databases. |
|
||||
| `POSTGRES_TUNE_CONNECTIONS` | `100` | PostgreSQL `max_connections`; automatically raised if Silo's app pool is configured higher. |
|
||||
| `POSTGRES_SHM_SIZE` | `8gb` | Docker `/dev/shm` size for the bundled PostgreSQL container. |
|
||||
|
||||
Advanced operators can still supply their own PostgreSQL configuration or
|
||||
override these env vars. Set `POSTGRES_TUNE=off` when you do not want Silo to
|
||||
change PostgreSQL server settings. Settings already written with `ALTER SYSTEM`
|
||||
remain in `postgresql.auto.conf`; reset those PostgreSQL parameters if you later
|
||||
move fully to a custom `postgresql.conf`.
|
||||
|
||||
## Project Structure
|
||||
|
||||
|
||||
+176
-15
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -30,6 +31,7 @@ import (
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/activitylog"
|
||||
"github.com/Silo-Server/silo-server/internal/adminjob"
|
||||
"github.com/Silo-Server/silo-server/internal/api"
|
||||
@@ -37,6 +39,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/audiobooks"
|
||||
"github.com/Silo-Server/silo-server/internal/audiobooks/podcastfeed"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/autoscan"
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/catalogseed"
|
||||
@@ -186,6 +189,70 @@ func configureOperationalLogging(
|
||||
return operationalWriter, opslog.NewRepo(pool), opsPM
|
||||
}
|
||||
|
||||
func maybeApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, appMaxConnections int, mode string) {
|
||||
switch strings.ToLower(strings.TrimSpace(mode)) {
|
||||
case "", "integrated", "api":
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
opts, err := database.LoadPostgresTuneOptionsFromEnv(appMaxConnections)
|
||||
if err != nil {
|
||||
slog.Warn("postgres auto-tuning disabled", "error", err)
|
||||
return
|
||||
}
|
||||
if !opts.Enabled {
|
||||
return
|
||||
}
|
||||
|
||||
tuneCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result, err := database.ApplyPostgresTuning(tuneCtx, pool, opts)
|
||||
for _, failure := range result.Failures {
|
||||
slog.Warn("postgres auto-tuning setting failed",
|
||||
"name", failure.Name,
|
||||
"value", failure.Value,
|
||||
"error", failure.Err,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
slog.Warn("postgres auto-tuning failed",
|
||||
"error", err,
|
||||
"applied", result.Applied,
|
||||
"failures", len(result.Failures),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("postgres auto-tuning applied",
|
||||
"profile", opts.Profile,
|
||||
"postgres_major", result.PostgresMajorVersion,
|
||||
"settings", result.Applied,
|
||||
"resets", len(result.Reset),
|
||||
"failures", len(result.Failures),
|
||||
"memory_budget_bytes", opts.MemoryBudgetBytes,
|
||||
"detected_memory_bytes", opts.DetectedMemoryBytes,
|
||||
"memory_source", opts.MemorySource,
|
||||
"memory_budget_percent", opts.MemoryBudgetPercent,
|
||||
"cpus", opts.CPUs,
|
||||
"connections", opts.Connections,
|
||||
"storage", opts.Storage,
|
||||
"db_size", result.DBSize,
|
||||
"database_size_bytes", result.DatabaseSizeBytes,
|
||||
)
|
||||
if len(result.RestartRequired) > 0 {
|
||||
slog.Warn("postgres restart required to finish applying auto-tuned settings",
|
||||
"settings", strings.Join(result.RestartRequired, ","),
|
||||
)
|
||||
}
|
||||
if len(result.Reset) > 0 {
|
||||
slog.Info("postgres auto-tuning reset stale settings",
|
||||
"settings", strings.Join(result.Reset, ","),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
envFile := flag.String("env", ".env", "path to .env bootstrap file")
|
||||
flag.Parse()
|
||||
@@ -316,6 +383,7 @@ func main() {
|
||||
slog.SetDefault(slog.New(logfilter.New(baseHandler, cfg.Server.LogQuiet)))
|
||||
|
||||
mode := cfg.Server.Mode
|
||||
maybeApplyPostgresTuning(ctx, pool, cfg.Database.MaxConnections, mode)
|
||||
slog.Info("silo starting", "mode", mode, "listen", cfg.Server.Listen, "log_level", cfg.Server.LogLevel, "node_id", nodeID)
|
||||
|
||||
appCtx, appCancel := context.WithCancel(ctx)
|
||||
@@ -428,6 +496,8 @@ func main() {
|
||||
audiobooksEnabled = false
|
||||
}
|
||||
deps.AudiobooksEnabled = audiobooksEnabled
|
||||
adminJobCancelRegistry := adminjob.NewCancelRegistry()
|
||||
deps.AdminJobCancelRegistry = adminJobCancelRegistry
|
||||
if needsWorkers && deps.DB != nil {
|
||||
deps.IntroRepository = intromarkers.NewRepository(deps.DB)
|
||||
deps.IntroAnalyzer = intromarkers.NewAnalyzer(
|
||||
@@ -735,6 +805,12 @@ func main() {
|
||||
pluginService.OnLifecycleChange(appCtx)
|
||||
}
|
||||
|
||||
// backgroundInit collects non-critical startup work (catalog-size-dependent
|
||||
// seeding, network-bound reconciliation) that must not block the HTTP
|
||||
// listener. The steps run sequentially in a background goroutine once the
|
||||
// server is ready to serve. Failures are logged, never fatal.
|
||||
var backgroundInit []func(context.Context)
|
||||
|
||||
// Step 4b: Create metadata service and match worker (if needed).
|
||||
var metadataService *metadata.MetadataService
|
||||
var personRefreshService *metadata.PersonRefreshService
|
||||
@@ -808,6 +884,8 @@ func main() {
|
||||
providerIDRepo := catalog.NewProviderIDRepository(deps.DB)
|
||||
movieQueueRepo = metadata.NewMovieMatchQueueRepository(deps.DB, deps.FileRepo)
|
||||
seriesQueueRepo = metadata.NewSeriesRootMatchQueueRepository(deps.DB)
|
||||
deps.MovieMatchQueueRepo = movieQueueRepo
|
||||
deps.SeriesRootMatchQueueRepo = seriesQueueRepo
|
||||
matchQueueCoordinator = metadata.NewMatchQueueCoordinator(movieQueueRepo, seriesQueueRepo)
|
||||
rootClaimRepo = catalog.NewRootClaimRepository(deps.DB)
|
||||
groupClaimRepo = catalog.NewGroupClaimRepository(deps.DB)
|
||||
@@ -857,27 +935,33 @@ func main() {
|
||||
}
|
||||
|
||||
matchWorker = metadata.NewMatchWorker(metadataService, deps.FileRepo, cfg.Matcher.Workers, cfg.Matcher.BatchSize, 30*time.Second)
|
||||
matchWorker.SetRealtimeHub(deps.RealtimeHub)
|
||||
if movieQueueRepo != nil {
|
||||
matchWorker.SetMovieFileClaimer(movieQueueRepo)
|
||||
}
|
||||
if seriesQueueRepo != nil {
|
||||
if cleaned, err := seriesQueueRepo.CleanupLegacySeriesGroupQueue(appCtx); err != nil {
|
||||
slog.Warn("failed to clean legacy series group queue rows", "error", err)
|
||||
} else if cleaned > 0 {
|
||||
slog.Info("cleaned legacy series group queue rows", "count", cleaned)
|
||||
}
|
||||
matchWorker.SetSeriesRootClaimer(seriesQueueRepo, cfg.Matcher.TVSeriesRootQueueEnabled())
|
||||
backgroundInit = append(backgroundInit, func(ctx context.Context) {
|
||||
if cleaned, err := seriesQueueRepo.CleanupLegacySeriesGroupQueue(ctx); err != nil {
|
||||
slog.Warn("failed to clean legacy series group queue rows", "error", err)
|
||||
} else if cleaned > 0 {
|
||||
slog.Info("cleaned legacy series group queue rows", "count", cleaned)
|
||||
}
|
||||
})
|
||||
}
|
||||
if deps.FolderRepo != nil {
|
||||
enabledFolders, err := deps.FolderRepo.GetEnabled(appCtx)
|
||||
if err != nil {
|
||||
slog.Warn("failed to seed metadata queues", "error", err)
|
||||
} else {
|
||||
backgroundInit = append(backgroundInit, func(ctx context.Context) {
|
||||
start := time.Now()
|
||||
enabledFolders, err := deps.FolderRepo.GetEnabled(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("failed to seed metadata queues", "error", err)
|
||||
return
|
||||
}
|
||||
seedMovieQueue := func(folderID int) {
|
||||
if movieQueueRepo == nil {
|
||||
return
|
||||
}
|
||||
if err := movieQueueRepo.SyncForFolder(appCtx, folderID); err != nil {
|
||||
if err := movieQueueRepo.SyncForFolder(ctx, folderID); err != nil {
|
||||
slog.Warn("failed to seed movie match queue", "folder_id", folderID, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -885,7 +969,7 @@ func main() {
|
||||
if seriesQueueRepo == nil {
|
||||
return
|
||||
}
|
||||
if err := seriesQueueRepo.SyncForFolder(appCtx, folderID); err != nil {
|
||||
if err := seriesQueueRepo.SyncForFolder(ctx, folderID); err != nil {
|
||||
slog.Warn("failed to seed series root queue", "folder_id", folderID, "error", err)
|
||||
}
|
||||
}
|
||||
@@ -903,7 +987,8 @@ func main() {
|
||||
seedMovieQueue(folder.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
slog.Info("deferred init: metadata match queues seeded", "folders", len(enabledFolders), "duration", time.Since(start))
|
||||
})
|
||||
}
|
||||
|
||||
deps.SkippedRootRepo = skippedRootRepo
|
||||
@@ -1042,9 +1127,11 @@ func main() {
|
||||
WithMatcher(historyimport.NewMatcher(historyRepo)).
|
||||
WithWatchState(watchstate.NewService(userStoreProvider).WithStableIdentityResolver(historyIdentity)).
|
||||
WithUserStoreProvider(userStoreProvider)
|
||||
if err := watchProviderService.SweepOpenScrobbles(appCtx); err != nil {
|
||||
slog.Warn("failed to sweep open watch provider scrobbles", "error", err)
|
||||
}
|
||||
backgroundInit = append(backgroundInit, func(ctx context.Context) {
|
||||
if err := watchProviderService.SweepOpenScrobbles(ctx); err != nil {
|
||||
slog.Warn("failed to sweep open watch provider scrobbles", "error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
deps.SessionMgr = sessionMgr
|
||||
deps.PlaybackRealtimeHub = playback.NewRealtimeHub()
|
||||
@@ -1250,6 +1337,7 @@ func main() {
|
||||
// Construct collection service for both the router and the collection sync scheduler.
|
||||
var collectionSyncScheduler *catalog.CollectionSyncScheduler
|
||||
var userCollectionScheduler *usercollections.Scheduler
|
||||
var trendingRefresher *sections.TrendingRefresher
|
||||
if needsWorkers && deps.DB != nil {
|
||||
collectionRepo := catalog.NewLibraryCollectionRepository(deps.DB)
|
||||
collItemRepo := catalog.NewItemRepository(deps.DB)
|
||||
@@ -1259,6 +1347,19 @@ func main() {
|
||||
deps.CollectionService = collectionService
|
||||
collectionSyncScheduler = catalog.NewCollectionSyncScheduler(collectionRepo, collectionService, slog.Default())
|
||||
|
||||
// The trending refresher reuses the section repo (to find used source/
|
||||
// window combos), a snapshot repo, an item repo (external-ID matching),
|
||||
// and the TMDB fetcher. The Trakt fetcher needs settingsRepo and is
|
||||
// propagated onto deps.TrendingRefresher later in router.go.
|
||||
trendingRefresher = sections.NewTrendingRefresher(
|
||||
sectionRepo,
|
||||
sections.NewTrendingSnapshotRepository(pool),
|
||||
catalog.NewItemRepository(deps.DB),
|
||||
collectionService.TMDBCollections,
|
||||
collectionService.TraktCollections,
|
||||
)
|
||||
deps.TrendingRefresher = trendingRefresher
|
||||
|
||||
if deps.UserStoreProvider != nil {
|
||||
userSync := usercollections.NewService(deps.UserStoreProvider, collItemRepo, libraryItemRepo, nil, slog.Default())
|
||||
userSync.TMDBCollections = collectionService.TMDBCollections
|
||||
@@ -1303,6 +1404,9 @@ func main() {
|
||||
if collectionSyncScheduler != nil {
|
||||
taskMgr.Register(tasks.NewSyncCollectionsTask(collectionSyncScheduler))
|
||||
}
|
||||
if trendingRefresher != nil {
|
||||
taskMgr.Register(tasks.NewRefreshTrendingDiscoverTask(trendingRefresher))
|
||||
}
|
||||
if userCollectionScheduler != nil {
|
||||
taskMgr.Register(tasks.NewSyncUserCollectionsTask(userCollectionScheduler))
|
||||
}
|
||||
@@ -1319,7 +1423,39 @@ func main() {
|
||||
)
|
||||
requestReconcileSvc.SetSecretResolver(settingsRepo)
|
||||
requestReconcileSvc.SetFulfillmentAdapters(radarr.NewClient(nil), sonarr.NewClient(nil))
|
||||
if userStoreProvider != nil {
|
||||
reconcileResolver := access.NewResolver(
|
||||
auth.NewUserRepository(deps.DB),
|
||||
userStoreProvider,
|
||||
access.NewProfileTokenService(cfg.Auth.JWTSecret, 0),
|
||||
)
|
||||
requestReconcileSvc.SetEntitlementResolver(mediarequests.NewAccessEntitlements(reconcileResolver))
|
||||
}
|
||||
taskMgr.Register(tasks.NewReconcileRequestsTask(requestReconcileSvc, 100))
|
||||
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil && pluginService != nil && pluginInstallationStore != nil {
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB)
|
||||
autoscanSvc := api.BuildAutoscanService(
|
||||
autoscanRepo,
|
||||
pluginService,
|
||||
pluginInstallationStore,
|
||||
mediarequests.NewRepository(deps.DB),
|
||||
settingsRepo,
|
||||
deps.FolderRepo,
|
||||
deps.LibraryScanQueue,
|
||||
deps.RedisClient,
|
||||
)
|
||||
// The poll task's default interval seeds the schedule from the stored
|
||||
// settings (DefaultPollIntervalSeconds); per-cycle gating still runs
|
||||
// off the live settings inside PollOnce. Seed in MILLISECONDS as
|
||||
// seconds*1000 — the SAME computation HandleUpdateSettings uses to
|
||||
// reschedule — so startup and reschedule agree for sub-minute and
|
||||
// non-60-multiple intervals (the old seconds/60 minutes path diverged).
|
||||
var intervalMs int64 = 10 * 60 * 1000
|
||||
if settings, serr := autoscanRepo.GetSettings(appCtx); serr == nil && settings.DefaultPollIntervalSeconds > 0 {
|
||||
intervalMs = int64(settings.DefaultPollIntervalSeconds) * 1000
|
||||
}
|
||||
taskMgr.Register(tasks.NewAutoscanPollTask(autoscanSvc, intervalMs))
|
||||
}
|
||||
reconcileProviderIDRepo := catalog.NewProviderIDRepository(deps.DB)
|
||||
reconcileEpisodeRepo := catalog.NewEpisodeRepository(deps.DB)
|
||||
historyResolver := watchstate.NewStableIdentityResolver(nil, reconcileEpisodeRepo, reconcileProviderIDRepo)
|
||||
@@ -1589,6 +1725,7 @@ func main() {
|
||||
templateBundleApplyExecutor,
|
||||
deps.RealtimeHub,
|
||||
)
|
||||
adminJobRunner.SetCancelRegistry(adminJobCancelRegistry)
|
||||
adminJobRunner.Start()
|
||||
defer adminJobRunner.Stop()
|
||||
|
||||
@@ -1766,6 +1903,30 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// Run non-critical startup work in the background so it doesn't delay the
|
||||
// HTTP listener from accepting connections. Steps run sequentially and stop
|
||||
// early if the app context is cancelled (shutdown).
|
||||
if len(backgroundInit) > 0 {
|
||||
go func() {
|
||||
start := time.Now()
|
||||
for _, step := range backgroundInit {
|
||||
if appCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
func() {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
slog.Error("deferred startup init step panicked; continuing",
|
||||
"panic", p, "stack", string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
step(appCtx)
|
||||
}()
|
||||
}
|
||||
slog.Info("deferred startup init completed", "steps", len(backgroundInit), "duration", time.Since(start))
|
||||
}()
|
||||
}
|
||||
|
||||
errCh := make(chan error, 3)
|
||||
go func() {
|
||||
slog.Info("HTTP server listening", "addr", cfg.Server.Listen)
|
||||
|
||||
@@ -132,6 +132,9 @@ func TestBuildLiveSessionSync_UsesTransportPlayMethod(t *testing.T) {
|
||||
RequestedMediaFileID: 41,
|
||||
PlayMethod: playback.PlayTranscode,
|
||||
BasePlayMethod: playback.PlayRemux,
|
||||
TranscodeHWAccel: "qsv",
|
||||
Position: 125.5,
|
||||
IsPaused: true,
|
||||
},
|
||||
want: "transcode",
|
||||
},
|
||||
@@ -164,6 +167,18 @@ func TestBuildLiveSessionSync_UsesTransportPlayMethod(t *testing.T) {
|
||||
if got.SessionID != tc.session.ID {
|
||||
t.Fatalf("SessionID = %q, want %q", got.SessionID, tc.session.ID)
|
||||
}
|
||||
if got.ProfileID != tc.session.ProfileID {
|
||||
t.Fatalf("ProfileID = %q, want %q", got.ProfileID, tc.session.ProfileID)
|
||||
}
|
||||
if got.PositionSeconds != tc.session.Position {
|
||||
t.Fatalf("PositionSeconds = %v, want %v", got.PositionSeconds, tc.session.Position)
|
||||
}
|
||||
if got.IsPaused != tc.session.IsPaused {
|
||||
t.Fatalf("IsPaused = %v, want %v", got.IsPaused, tc.session.IsPaused)
|
||||
}
|
||||
if got.TranscodeHWAccel != tc.session.TranscodeHWAccel {
|
||||
t.Fatalf("TranscodeHWAccel = %q, want %q", got.TranscodeHWAccel, tc.session.TranscodeHWAccel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,10 @@ func buildLiveSessionSync(s *playback.Session, reportingNode string) worker.Sess
|
||||
TargetVideoCodec: s.TargetVideoCodec,
|
||||
TargetAudioCodec: s.TargetAudioCodec,
|
||||
TargetBitrateKbps: s.TargetBitrateKbps,
|
||||
TranscodeHWAccel: s.TranscodeHWAccel,
|
||||
StartedAt: s.StartedAt,
|
||||
UpdatedAt: s.UpdatedAt,
|
||||
PositionSeconds: s.Position,
|
||||
IsPaused: s.IsPaused,
|
||||
HasWebSocket: s.HasWebSocket,
|
||||
}
|
||||
|
||||
+6
-3
@@ -2,7 +2,7 @@ services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
restart: unless-stopped
|
||||
shm_size: 1gb
|
||||
shm_size: ${POSTGRES_SHM_SIZE:-8gb}
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-silo}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-silo}
|
||||
@@ -11,8 +11,7 @@ services:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/postgres:/var/lib/postgresql
|
||||
- ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
||||
command: ["postgres", "-c", "listen_addresses=*"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-silo}"]
|
||||
interval: 5s
|
||||
@@ -40,6 +39,7 @@ services:
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins
|
||||
POSTGRES_TUNE: ${POSTGRES_TUNE:-auto}
|
||||
ports:
|
||||
- "${PORT:-8090}:8080"
|
||||
- "${JF_PORT:-8096}:8096"
|
||||
@@ -51,6 +51,7 @@ services:
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/audiobook-covers:/var/lib/silo/audiobook-covers
|
||||
- /proc/meminfo:/host/proc/meminfo:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -59,6 +60,8 @@ services:
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
|
||||
# Optional distributed-mode example. Most single-host installs should
|
||||
# just use the integrated service above.
|
||||
|
||||
@@ -0,0 +1,724 @@
|
||||
# Resilient Library Deletion Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Rewrite `delete_library` so it deletes a library in small, autocommitted, deadlock-retrying batches instead of one multi-minute transaction that deadlocks and rolls back on large libraries.
|
||||
|
||||
**Architecture:** `FolderRepository.DeleteWithStats` becomes a phased, batched operation (orphan items → folder media_files → remaining shared junctions → folder row). Each batch is one autocommit statement wrapped in a `retryOnDeadlock` helper, so locks are held briefly and the job is resumable. Two small helpers (`retryOnDeadlock`, `deleteInBatches`) are unit-tested in pure Go; the full path is verified against the live dev database.
|
||||
|
||||
**Tech Stack:** Go, pgx v5 (`pgxpool`, `pgconn`), PostgreSQL. Spec: `docs/superpowers/specs/2026-05-28-library-delete-resilience-design.md`.
|
||||
|
||||
All commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- **Modify** `internal/catalog/folder_repo.go`
|
||||
- Add constants `orphanDeleteBatch`, `folderChildDeleteBatch`.
|
||||
- Add package vars `deadlockMaxAttempts`, `deadlockBaseBackoff` and func `retryOnDeadlock`.
|
||||
- Add func `deleteInBatches`.
|
||||
- Add interface `rowQuerier`; change `collectImageDirs` and `filterUnreferencedImageDirs` to take `rowQuerier` instead of `pgx.Tx`; extract `collectRawImageDirs`.
|
||||
- Rewrite `DeleteWithStats`; add helpers `collectOrphanBatch`, `dirSetToSlice`.
|
||||
- Leave `collectOrphanIDs` and `Delete` unchanged.
|
||||
- **Create** `internal/catalog/folder_delete_test.go` — pure-Go unit tests for `retryOnDeadlock` and `deleteInBatches`.
|
||||
- **Do not modify** `internal/catalog/library_repo.go` — its `collectImageDirs`/`collectOrphanIDs` calls keep compiling because `pgx.Tx` satisfies `rowQuerier`. (Verified by build.)
|
||||
- **Unchanged behavior** in `internal/adminjob/library_delete.go` (executor), `internal/adminjob/runner.go` (heartbeat).
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Deadlock-retry helper
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go`
|
||||
- Test: `internal/catalog/folder_delete_test.go` (create)
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Create `internal/catalog/folder_delete_test.go`:
|
||||
|
||||
```go
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// withFastDeadlockRetry shrinks retry timing/attempts for tests and restores
|
||||
// the originals on cleanup. Tests using it must not call t.Parallel().
|
||||
func withFastDeadlockRetry(t *testing.T, maxAttempts int) {
|
||||
t.Helper()
|
||||
oldMax, oldBackoff := deadlockMaxAttempts, deadlockBaseBackoff
|
||||
deadlockMaxAttempts = maxAttempts
|
||||
deadlockBaseBackoff = time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
deadlockMaxAttempts = oldMax
|
||||
deadlockBaseBackoff = oldBackoff
|
||||
})
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockRetriesThenSucceeds(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
calls := 0
|
||||
err := retryOnDeadlock(context.Background(), func() error {
|
||||
calls++
|
||||
if calls < 3 {
|
||||
return &pgconn.PgError{Code: "40P01"}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected success, got %v", err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("expected 3 calls, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockReturnsNonRetryableImmediately(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
sentinel := errors.New("boom")
|
||||
calls := 0
|
||||
err := retryOnDeadlock(context.Background(), func() error {
|
||||
calls++
|
||||
return sentinel
|
||||
})
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("expected sentinel, got %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockGivesUpAfterMaxAttempts(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 4)
|
||||
calls := 0
|
||||
err := retryOnDeadlock(context.Background(), func() error {
|
||||
calls++
|
||||
return &pgconn.PgError{Code: "40P01"}
|
||||
})
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) || pgErr.Code != "40P01" {
|
||||
t.Fatalf("expected 40P01 pg error, got %v", err)
|
||||
}
|
||||
if calls != 4 {
|
||||
t.Fatalf("expected 4 calls, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockStopsOnCanceledContext(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
calls := 0
|
||||
err := retryOnDeadlock(ctx, func() error {
|
||||
calls++
|
||||
return &pgconn.PgError{Code: "40P01"}
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled, got %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("expected 1 call before cancel, got %d", calls)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestRetryOnDeadlock -v`
|
||||
Expected: compile failure / FAIL — `undefined: retryOnDeadlock`, `deadlockMaxAttempts`, `deadlockBaseBackoff`.
|
||||
|
||||
- [ ] **Step 3: Implement the helper**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add near the top of the file (after the `import` block, before the sentinel errors var is fine):
|
||||
|
||||
```go
|
||||
// Retry parameters for transient serialization/deadlock failures. They are
|
||||
// package vars (not consts) only so tests can shrink them; production code
|
||||
// never mutates them.
|
||||
var (
|
||||
deadlockMaxAttempts = 5
|
||||
deadlockBaseBackoff = 50 * time.Millisecond
|
||||
)
|
||||
|
||||
// retryOnDeadlock runs op, retrying when Postgres reports a deadlock (40P01) or
|
||||
// serialization failure (40001), with exponential backoff. It returns
|
||||
// immediately for any other error, and honors context cancellation between
|
||||
// attempts.
|
||||
func retryOnDeadlock(ctx context.Context, op func() error) error {
|
||||
backoff := deadlockBaseBackoff
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := op()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if attempt < deadlockMaxAttempts && errors.As(err, &pgErr) &&
|
||||
(pgErr.Code == "40P01" || pgErr.Code == "40001") {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
backoff *= 2
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
(`context`, `errors`, `time`, and `github.com/jackc/pgx/v5/pgconn` are already imported in this file.)
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestRetryOnDeadlock -v`
|
||||
Expected: PASS (4 tests).
|
||||
|
||||
- [ ] **Step 5: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git add internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git commit -m "feat(catalog): add deadlock-retry helper for batched deletes"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Batched-delete loop helper
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go`
|
||||
- Test: `internal/catalog/folder_delete_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Append to `internal/catalog/folder_delete_test.go`:
|
||||
|
||||
```go
|
||||
func TestDeleteInBatchesLoopsUntilUnderBatchSize(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
counts := []int64{5, 5, 2}
|
||||
idx := 0
|
||||
total, err := deleteInBatches(context.Background(), 5, func(context.Context) (int64, error) {
|
||||
n := counts[idx]
|
||||
idx++
|
||||
return n, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if total != 12 {
|
||||
t.Fatalf("expected total 12, got %d", total)
|
||||
}
|
||||
if idx != 3 {
|
||||
t.Fatalf("expected 3 batches, got %d", idx)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteInBatchesStopsImmediatelyWhenFirstBatchUnderSize(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
calls := 0
|
||||
total, err := deleteInBatches(context.Background(), 5, func(context.Context) (int64, error) {
|
||||
calls++
|
||||
return 0, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if total != 0 || calls != 1 {
|
||||
t.Fatalf("expected total 0 and 1 call, got total=%d calls=%d", total, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteInBatchesReturnsError(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
sentinel := errors.New("delete failed")
|
||||
_, err := deleteInBatches(context.Background(), 5, func(context.Context) (int64, error) {
|
||||
return 0, sentinel
|
||||
})
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("expected sentinel, got %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestDeleteInBatches -v`
|
||||
Expected: FAIL — `undefined: deleteInBatches`.
|
||||
|
||||
- [ ] **Step 3: Implement the helper**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add after `retryOnDeadlock`:
|
||||
|
||||
```go
|
||||
// deleteInBatches repeatedly runs deleteBatch (each a single autocommit
|
||||
// statement) until a batch removes fewer than batchSize rows. Each batch is
|
||||
// retried on deadlock. It returns the total number of rows deleted.
|
||||
func deleteInBatches(
|
||||
ctx context.Context,
|
||||
batchSize int,
|
||||
deleteBatch func(ctx context.Context) (int64, error),
|
||||
) (int64, error) {
|
||||
var total int64
|
||||
for {
|
||||
var affected int64
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
n, e := deleteBatch(ctx)
|
||||
affected = n
|
||||
return e
|
||||
}); err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += affected
|
||||
if affected < int64(batchSize) {
|
||||
return total, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestDeleteInBatches -v`
|
||||
Expected: PASS (3 tests).
|
||||
|
||||
- [ ] **Step 5: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git add internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git commit -m "feat(catalog): add deleteInBatches loop helper"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Generalize image-dir helpers to a querier interface
|
||||
|
||||
This lets the image-dir helpers run on either a `*pgxpool.Pool` (new batched path) or a `pgx.Tx` (existing `library_repo.go` path), and splits raw collection from filtering so the batched path can collect cheaply per batch and filter once.
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go:547-635` (the `collectImageDirs` / `filterUnreferencedImageDirs` block)
|
||||
|
||||
- [ ] **Step 1: Add the `rowQuerier` interface**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add (near the other type declarations, e.g. just above `collectOrphanIDs`):
|
||||
|
||||
```go
|
||||
// rowQuerier is satisfied by both *pgxpool.Pool and pgx.Tx, letting read
|
||||
// helpers run inside or outside an explicit transaction.
|
||||
type rowQuerier interface {
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Split `collectImageDirs` into raw collection + filtering**
|
||||
|
||||
Replace the existing `collectImageDirs` function (currently `internal/catalog/folder_repo.go:549-584`) with these two functions:
|
||||
|
||||
```go
|
||||
// collectImageDirs returns S3 directory prefixes for images belonging to the
|
||||
// given content IDs that are not still referenced by other surviving content.
|
||||
func collectImageDirs(ctx context.Context, q rowQuerier, contentIDs []string) ([]string, error) {
|
||||
dirs, err := collectRawImageDirs(ctx, q, contentIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return filterUnreferencedImageDirs(ctx, q, dirs, contentIDs)
|
||||
}
|
||||
|
||||
// collectRawImageDirs returns the deduped S3 directory prefixes referenced by
|
||||
// the given content IDs (items, their seasons, and their episodes), without
|
||||
// filtering out dirs still used by other content.
|
||||
func collectRawImageDirs(ctx context.Context, q rowQuerier, contentIDs []string) ([]string, error) {
|
||||
imgRows, err := q.Query(ctx, `
|
||||
SELECT poster_path, backdrop_path, logo_path FROM media_items WHERE content_id = ANY($1)
|
||||
UNION ALL
|
||||
SELECT poster_path, '', '' FROM seasons WHERE series_id = ANY($1)
|
||||
UNION ALL
|
||||
SELECT still_path, '', '' FROM episodes WHERE series_id = ANY($1)
|
||||
`, contentIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("collecting image paths: %w", err)
|
||||
}
|
||||
defer imgRows.Close()
|
||||
dirSet := make(map[string]struct{})
|
||||
for imgRows.Next() {
|
||||
var p1, p2, p3 string
|
||||
if err := imgRows.Scan(&p1, &p2, &p3); err != nil {
|
||||
return nil, fmt.Errorf("scanning image path: %w", err)
|
||||
}
|
||||
for _, p := range []string{p1, p2, p3} {
|
||||
if p != "" && !strings.Contains(p, "://") {
|
||||
if dir := pathDir(p); dir != "" {
|
||||
dirSet[dir] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := imgRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating image paths: %w", err)
|
||||
}
|
||||
dirs := make([]string, 0, len(dirSet))
|
||||
for dir := range dirSet {
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
return dirs, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Change `filterUnreferencedImageDirs` to take `rowQuerier`**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, change the signature (currently `internal/catalog/folder_repo.go:586`) from:
|
||||
|
||||
```go
|
||||
func filterUnreferencedImageDirs(ctx context.Context, tx pgx.Tx, dirs, deletingContentIDs []string) ([]string, error) {
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```go
|
||||
func filterUnreferencedImageDirs(ctx context.Context, q rowQuerier, dirs, deletingContentIDs []string) ([]string, error) {
|
||||
```
|
||||
|
||||
and change the single `tx.Query(ctx, ...)` call inside it to `q.Query(ctx, ...)`. Leave the SQL and the rest of the body unchanged.
|
||||
|
||||
- [ ] **Step 4: Verify the build (proves `library_repo.go` still compiles with `pgx.Tx`)**
|
||||
|
||||
Run: `go build ./... && go vet ./internal/catalog/`
|
||||
Expected: no errors. (`library_repo.go` passes a `pgx.Tx` to `collectImageDirs`, which now accepts `rowQuerier`; `pgx.Tx` satisfies it.)
|
||||
|
||||
- [ ] **Step 5: Run the catalog tests**
|
||||
|
||||
Run: `go test ./internal/catalog/...`
|
||||
Expected: PASS (existing tests plus Tasks 1–2 helpers).
|
||||
|
||||
- [ ] **Step 6: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go
|
||||
git add internal/catalog/folder_repo.go
|
||||
git commit -m "refactor(catalog): make image-dir helpers querier-agnostic"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Rewrite `DeleteWithStats` as a batched, phased operation
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go:415-511` (the `DeleteWithStats` body) and add two helpers + two constants.
|
||||
|
||||
- [ ] **Step 1: Add the batch-size constants**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add near the retry vars from Task 1:
|
||||
|
||||
```go
|
||||
const (
|
||||
// orphanDeleteBatch is small because each media_items row cascades across
|
||||
// ~15 child tables.
|
||||
orphanDeleteBatch = 1000
|
||||
// folderChildDeleteBatch covers the lighter folder-scoped media_files and
|
||||
// junction deletes.
|
||||
folderChildDeleteBatch = 5000
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Replace the `DeleteWithStats` body**
|
||||
|
||||
Replace the entire existing `DeleteWithStats` function (`internal/catalog/folder_repo.go:415-511`) with:
|
||||
|
||||
```go
|
||||
func (r *FolderRepository) DeleteWithStats(
|
||||
ctx context.Context,
|
||||
id int,
|
||||
progress func(current, total int, message string),
|
||||
) (*DeleteFolderStats, error) {
|
||||
stats := &DeleteFolderStats{}
|
||||
|
||||
// Phase 0: preflight reads (no long-lived transaction).
|
||||
if err := r.pool.QueryRow(ctx, `SELECT name FROM media_folders WHERE id = $1`, id).Scan(&stats.LibraryName); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrFolderNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("loading folder before delete: %w", err)
|
||||
}
|
||||
if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM media_files WHERE media_folder_id = $1`, id).Scan(&stats.MediaFiles); err != nil {
|
||||
return nil, fmt.Errorf("counting media files: %w", err)
|
||||
}
|
||||
if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = $1`, id).Scan(&stats.MediaItemLinks); err != nil {
|
||||
return nil, fmt.Errorf("counting media item links: %w", err)
|
||||
}
|
||||
var orphanTotal int
|
||||
if err := r.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.media_folder_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries other
|
||||
WHERE other.content_id = mil.content_id
|
||||
AND other.media_folder_id <> $1
|
||||
)`, id).Scan(&orphanTotal); err != nil {
|
||||
return nil, fmt.Errorf("counting orphaned items: %w", err)
|
||||
}
|
||||
|
||||
// Phase 1: delete orphaned media_items in detect-then-delete batches.
|
||||
// Cascade removes their junctions, provider IDs, episodes, seasons, etc.
|
||||
if progress != nil {
|
||||
progress(0, orphanTotal, "Deleting orphaned items")
|
||||
}
|
||||
rawDirs := make(map[string]struct{})
|
||||
for {
|
||||
ids, err := r.collectOrphanBatch(ctx, id, orphanDeleteBatch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
dirs, err := collectRawImageDirs(ctx, r.pool, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, d := range dirs {
|
||||
rawDirs[d] = struct{}{}
|
||||
}
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
_, e := r.pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, ids)
|
||||
return e
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting orphaned items: %w", err)
|
||||
}
|
||||
stats.OrphanedItems += len(ids)
|
||||
if progress != nil {
|
||||
progress(stats.OrphanedItems, orphanTotal, "Deleting orphaned items")
|
||||
}
|
||||
}
|
||||
|
||||
// Filter accumulated image dirs once, now that orphans are gone, against
|
||||
// any surviving content. Empty deleting-set means "exclude nothing".
|
||||
if len(rawDirs) > 0 {
|
||||
filtered, err := filterUnreferencedImageDirs(ctx, r.pool, dirSetToSlice(rawDirs), []string{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats.OrphanedImageDirs = filtered
|
||||
}
|
||||
|
||||
// Phase 2: delete this folder's media_files (folder-tied, not item-tied).
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Deleting media files")
|
||||
}
|
||||
if _, err := deleteInBatches(ctx, folderChildDeleteBatch, func(ctx context.Context) (int64, error) {
|
||||
tag, e := r.pool.Exec(ctx, `
|
||||
DELETE FROM media_files
|
||||
WHERE id IN (
|
||||
SELECT id FROM media_files WHERE media_folder_id = $1 LIMIT $2
|
||||
)`, id, folderChildDeleteBatch)
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting media files: %w", err)
|
||||
}
|
||||
|
||||
// Phase 3: delete remaining folder memberships (shared items kept; only the
|
||||
// membership in this folder is removed).
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Removing library memberships")
|
||||
}
|
||||
if _, err := deleteInBatches(ctx, folderChildDeleteBatch, func(ctx context.Context) (int64, error) {
|
||||
tag, e := r.pool.Exec(ctx, `
|
||||
DELETE FROM media_item_libraries
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM media_item_libraries WHERE media_folder_id = $1 LIMIT $2
|
||||
)`, id, folderChildDeleteBatch)
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting media item links: %w", err)
|
||||
}
|
||||
|
||||
// Phase 4: delete the now-lightweight folder row. Tolerate 0 rows so a
|
||||
// resumed run that already removed it still succeeds.
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
_, e := r.pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, id)
|
||||
return e
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting folder: %w", err)
|
||||
}
|
||||
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Library deletion completed")
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// collectOrphanBatch returns up to limit content IDs whose only library
|
||||
// membership is the given folder.
|
||||
func (r *FolderRepository) collectOrphanBatch(ctx context.Context, folderID, limit int) ([]string, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT mil.content_id
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.media_folder_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries other
|
||||
WHERE other.content_id = mil.content_id
|
||||
AND other.media_folder_id <> $1
|
||||
)
|
||||
LIMIT $2`, folderID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("finding orphaned items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var contentID string
|
||||
if err := rows.Scan(&contentID); err != nil {
|
||||
return nil, fmt.Errorf("scanning orphan content_id: %w", err)
|
||||
}
|
||||
ids = append(ids, contentID)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
// dirSetToSlice returns the keys of set as a slice (nil if empty).
|
||||
func dirSetToSlice(set map[string]struct{}) []string {
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
dirs := make([]string, 0, len(set))
|
||||
for d := range set {
|
||||
dirs = append(dirs, d)
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
```
|
||||
|
||||
Note: this removes the old single-transaction body (`tx, err := r.pool.Begin(...)`, the inline orphan-collection loop, the `collectImageDirs` call, the CASCADE folder drop, and `tx.Commit`). `collectOrphanIDs` and `Delete` are untouched. `collectImageDirs` remains (still used by `library_repo.go`).
|
||||
|
||||
- [ ] **Step 3: Verify the build and vet**
|
||||
|
||||
Run: `go build ./... && go vet ./internal/catalog/`
|
||||
Expected: no errors. If `go vet` flags an unused function, confirm it is genuinely unused before removing — `collectImageDirs`, `collectOrphanIDs`, and `pathDir` must all remain (still referenced).
|
||||
|
||||
- [ ] **Step 4: Run the full catalog + adminjob test suites**
|
||||
|
||||
Run: `go test ./internal/catalog/... ./internal/adminjob/...`
|
||||
Expected: PASS. (No DB-backed test exercises `DeleteWithStats`; the helper unit tests and existing tests must stay green.)
|
||||
|
||||
- [ ] **Step 5: Lint**
|
||||
|
||||
Run: `golangci-lint run ./internal/catalog/...`
|
||||
Expected: no findings. (If `golangci-lint` is unavailable, run `gofmt -l internal/catalog/` and ensure it prints nothing.)
|
||||
|
||||
- [ ] **Step 6: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go
|
||||
git add internal/catalog/folder_repo.go
|
||||
git commit -m "fix(catalog): delete libraries in deadlock-retrying batches
|
||||
|
||||
Replaces the single multi-minute delete transaction with phased, batched
|
||||
autocommit deletes (orphan items, media files, memberships, folder row),
|
||||
each retried on deadlock. Holds only short locks, survives concurrent
|
||||
writers, and is resumable on failure."
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Verify on the dev server
|
||||
|
||||
This is the integration test (no catalog DB harness exists). Follow the deployment-debugging runbook. The Audiobooks library (folder id 8) is currently stuck with ~249K items / ~339K files.
|
||||
|
||||
- [ ] **Step 1: Deploy the branch to dev**
|
||||
|
||||
Run: `make dev-deploy`
|
||||
Expected: builds `silo:dev-local`, recreates the `silo` service. Then confirm health:
|
||||
`ssh "$DEV_HOST" 'curl -s http://localhost:8090/api/v1/ready'`
|
||||
|
||||
- [ ] **Step 2: Capture the starting counts**
|
||||
|
||||
Run (psql via the dev compose `postgres` service, user/db `continuum`):
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM media_files WHERE media_folder_id = 8) AS files,
|
||||
(SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = 8) AS links,
|
||||
(SELECT EXISTS (SELECT 1 FROM media_folders WHERE id = 8)) AS folder_exists;
|
||||
```
|
||||
|
||||
Expected: non-zero files/links, `folder_exists = t`.
|
||||
|
||||
- [ ] **Step 3: Trigger the deletion**
|
||||
|
||||
Re-trigger the library deletion through the normal path (admin UI "delete library" for Audiobooks, or the admin delete endpoint). Confirm a new job is queued:
|
||||
|
||||
```sql
|
||||
SELECT id, job_type, status, requested_at
|
||||
FROM admin_jobs WHERE job_type = 'delete_library'
|
||||
ORDER BY requested_at DESC LIMIT 3;
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Monitor to completion**
|
||||
|
||||
Watch the job and the draining counts (re-run periodically):
|
||||
|
||||
```sql
|
||||
SELECT id, status, progress_current, progress_total, error_message, heartbeat_at
|
||||
FROM admin_jobs WHERE job_type = 'delete_library'
|
||||
ORDER BY requested_at DESC LIMIT 1;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = 8;
|
||||
```
|
||||
|
||||
Also confirm deadlock retries (if any) are recovered rather than fatal:
|
||||
`ssh "$DEV_HOST" "$DEV_COMPOSE_CMD logs postgres 2>&1 | grep -i deadlock | tail -20"`
|
||||
Expected: the job reaches `status = completed`; counts trend to 0; any deadlocks are transient (the job does not fail).
|
||||
|
||||
- [ ] **Step 5: Confirm full removal**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM media_files WHERE media_folder_id = 8) AS files,
|
||||
(SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = 8) AS links,
|
||||
(SELECT EXISTS (SELECT 1 FROM media_folders WHERE id = 8)) AS folder_exists;
|
||||
```
|
||||
|
||||
Expected: `files = 0`, `links = 0`, `folder_exists = f`.
|
||||
|
||||
- [ ] **Step 6: Report**
|
||||
|
||||
Summarize: starting counts, time to completion, number of batches/retries observed, and final state. No commit (verification only).
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:**
|
||||
- Phased batched deletion (Phases 0–4) → Task 4. ✔
|
||||
- `retryOnDeadlock` (40P01/40001, 5 attempts, backoff, ctx-aware) → Task 1. ✔
|
||||
- Batch sizes `orphanDeleteBatch=1000`, `folderChildDeleteBatch=5000` → Task 4 Step 1. ✔
|
||||
- Autocommit per batch / no long transaction → Task 4 Step 2 (uses `r.pool.Exec`, no `Begin`). ✔
|
||||
- Stats + progress + heartbeat reliance → Task 4 Step 2; runner heartbeat unchanged. ✔
|
||||
- Resumability (folder row last, tolerate 0 rows) → Task 4 Phase 4. ✔
|
||||
- Unit tests for retry + batch loop → Tasks 1–2. ✔
|
||||
- `collectImageDirs` shared with `library_repo.go` preserved → Task 3 (interface widening), Task 4 Step 3 build check. ✔
|
||||
- Dev integration verification → Task 5. ✔
|
||||
- Out of scope (writer coordination, runner/executor changes, schema changes) → not present in any task. ✔
|
||||
|
||||
**Placeholder scan:** No TBD/TODO; every code step has complete code; every command has expected output. ✔
|
||||
|
||||
**Type consistency:** `retryOnDeadlock(ctx, func() error) error`, `deleteInBatches(ctx, int, func(ctx) (int64,error)) (int64,error)`, `rowQuerier{ Query(ctx,string,...any)(pgx.Rows,error) }`, `collectRawImageDirs`/`collectImageDirs`/`filterUnreferencedImageDirs(ctx, rowQuerier, ...)`, `collectOrphanBatch(ctx,int,int)([]string,error)`, `dirSetToSlice(map[string]struct{})[]string`, consts `orphanDeleteBatch`/`folderChildDeleteBatch`, vars `deadlockMaxAttempts`/`deadlockBaseBackoff` — names used consistently across tasks. ✔
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,779 @@
|
||||
# Autoscan Host Backend Implementation Plan (Part 1 of 2)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build the silo-server backend for the standalone Autoscan category: poll `scan_source.v1` plugins on a timer, resolve their Silo-native paths to library folders, and enqueue rescans — with connections decoupled from Requests.
|
||||
|
||||
**Architecture:** Mirror the existing `scheduled_task.v1` host dispatch (`pluginhost.Client` capability wrapper + `plugins.Service` resolver + a `taskmanager` task). Generalize the closed PR #43 autoscan engine (resolve → suppress → enqueue) so its input is "Silo-native paths from a scan-source plugin" instead of "arr history." Replace the old `request_integrations`-coupled schema with `autoscan_connections` + `autoscan_sources`.
|
||||
|
||||
**Tech Stack:** Go, pgx/v5, the `internal/plugins`/`internal/pluginhost` runtime, `internal/scantrigger`/`internal/scanqueue`, `internal/taskmanager`, numbered SQL migrations, `github.com/Silo-Server/silo-plugin-sdk` (the new `scan_source.v1` capability).
|
||||
|
||||
Commands assume the silo-server repository root is the cwd. This is **Part 1 (backend)**; the Autoscan admin UI is Part 2.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **SDK dependency.** This plan needs the SDK's `scan_source.v1` symbols (`runtime.Client.ScanSource()`, `pluginv1.ScanSourceClient`, `pluginv1.PollChangesRequest/Response`, `capability.ScanSource`). They are in `silo-plugin-sdk` PR #2 but **not yet released**. Until `v0.5.0` is tagged, add a local replace directive so the host builds against the working checkout:
|
||||
|
||||
```bash
|
||||
# from the repository root (cwd)
|
||||
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
This points the replace at the `silo-plugin-sdk` checkout (a sibling of this repo).
|
||||
|
||||
The final task removes the replace and bumps to the tagged version.
|
||||
|
||||
2. **Closed PR #43 is on `main`.** The in-process arr autoscan (`internal/autoscan/*`, `internal/api/handlers/autoscan.go`, `internal/taskmanager/tasks/autoscan_poll.go`, router routes, migration `171_autoscan`, the `AdminRequests.tsx` tab) is the salvage source. This plan **generalizes the engine and replaces the schema/wiring**; it does not preserve the arr-specific history client (that moves to the arr plugin in a separate plan).
|
||||
|
||||
3. **Tests that touch `internal/api/handlers` or `internal/api` link libvips via CGO.** Run the suite in the libvips-equipped container (see project memory `silo-test-libvips-gotcha`), not the bare host, or those packages report `[build failed]` and skip silently.
|
||||
|
||||
---
|
||||
|
||||
## Pattern references (read before starting)
|
||||
|
||||
- Host capability client wrapper: `internal/pluginhost/client.go` — `ScheduledTaskClient` type (l.42), `Client.ScheduledTask` accessor (l.111), `requireCapability` (l.157), `Run` (l.226).
|
||||
- Host resolver: `internal/plugins/service.go:413` `ScheduledTaskClient(ctx, installationID, capabilityID)` → `ensureClient` → `client.ScheduledTask(capabilityID)`.
|
||||
- Timer task over plugin capabilities: `internal/plugins/task_registry.go` (iterate enabled installations, filter capability type, build a `taskmanager.Task`).
|
||||
- Salvageable engine (generalize, don't rewrite): `internal/autoscan/service.go` (resolve/suppress/enqueue loop), `dedupe.go` (`uniqueParentDirs`), `suppress.go` (`Suppressor`). Discard `history.go`, `rewrite.go`, `suggest*.go` (arr-specific → arr plugin).
|
||||
- Migration mechanics: paired `migrations/NNN_*.up.sql`/`.down.sql`, embedded via `migrations/embed.go`, auto-applied on boot.
|
||||
- Credential decryption for the reuse-from-Requests case: `internal/requests/repository.go` (`integrationColumns` has `base_url, api_key_ref`); Fernet resolution via `catalog.ServerSettingsRepo.Get` (the same path PR #43's `SecretResolver` used).
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `internal/pluginhost/client.go` | `ScanSourceClient` wrapper + `Client.ScanSource()` accessor | Modify |
|
||||
| `internal/pluginhost/client_test.go` | `requireCapability` gate for `scan_source.v1` | Modify |
|
||||
| `internal/plugins/service.go` | `ScanSourceClient(ctx, installationID, capabilityID)` resolver | Modify |
|
||||
| `migrations/172_autoscan_v2.up.sql` / `.down.sql` | `autoscan_connections` + `autoscan_sources`; drop the `171` tables | Create |
|
||||
| `internal/autoscan/types.go` | `Source`, `Connection`, `Settings`, `PollChanges` provider seam | Rewrite |
|
||||
| `internal/autoscan/repository.go` | CRUD for connections + sources; `AdvanceMarker` | Rewrite |
|
||||
| `internal/autoscan/provider.go` | `ScanSourceProvider` interface + adapter over the plugins resolver | Create |
|
||||
| `internal/autoscan/connection.go` | Resolve a `Connection` (own or Requests-linked) → `{base_url, api_key}` | Create |
|
||||
| `internal/autoscan/service.go` | Generic engine: per source → `PollChanges(marker)` → resolve → suppress → enqueue → store marker | Rewrite |
|
||||
| `internal/autoscan/dedupe.go`, `suppress.go` | Unchanged salvage | Keep |
|
||||
| `internal/autoscan/history.go`, `rewrite.go`, `suggest*.go` | Arr-specific | Delete |
|
||||
| `internal/taskmanager/tasks/autoscan_poll.go` | Poll task → `Service.PollOnce` | Keep (retarget) |
|
||||
| `internal/api/handlers/autoscan.go` | Admin API (settings/connections/sources/trigger/status) | Rewrite |
|
||||
| `internal/api/router.go`, `cmd/silo/main.go` | Wire the rebuilt service + connection resolver; drop request-coupled wiring | Modify |
|
||||
| `go.mod` | SDK replace (dev) → `v0.5.0` (final) | Modify |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Host `pluginhost` client wrapper for `scan_source.v1`
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/pluginhost/client.go`
|
||||
- Modify: `internal/pluginhost/client_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
In `internal/pluginhost/client_test.go`, add a test mirroring the existing `scheduled_task` capability-gate test (find it first; it constructs a `Client` with a manifest declaring a capability and asserts the accessor errors when the capability is absent and succeeds when present):
|
||||
|
||||
```go
|
||||
func TestClientScanSourceRequiresCapability(t *testing.T) {
|
||||
// Client whose manifest does NOT declare scan_source.v1
|
||||
c := newTestClientWithCapabilities(t, "metadata_provider.v1")
|
||||
if _, err := c.ScanSource("missing"); err == nil {
|
||||
t.Fatal("ScanSource: expected error when capability absent")
|
||||
}
|
||||
// Client WHOSE manifest declares scan_source.v1 with id "arr"
|
||||
c2 := newTestClientWithCapabilities(t, "scan_source.v1:arr")
|
||||
if _, err := c2.ScanSource("arr"); err != nil {
|
||||
t.Fatalf("ScanSource: unexpected error when capability present: %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use whatever helper the sibling `scheduled_task` test uses to construct a `Client` with declared capabilities; mirror it exactly (do not invent a helper that does not exist — read `client_test.go` and copy the construction).
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `go test ./internal/pluginhost/ -run TestClientScanSourceRequiresCapability`
|
||||
Expected: FAIL — `c.ScanSource undefined`.
|
||||
|
||||
- [ ] **Step 3: Add the wrapper type and accessor**
|
||||
|
||||
In `internal/pluginhost/client.go`, after the `ScheduledTaskClient` type (l.42), add:
|
||||
|
||||
```go
|
||||
type ScanSourceClient struct {
|
||||
client pluginv1.ScanSourceClient
|
||||
timeout time.Duration
|
||||
}
|
||||
```
|
||||
|
||||
After the `Client.ScheduledTask` accessor (l.117), add:
|
||||
|
||||
```go
|
||||
func (c *Client) ScanSource(capabilityID string) (*ScanSourceClient, error) {
|
||||
if err := c.requireCapability("scan_source.v1", capabilityID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ScanSourceClient{
|
||||
client: c.rpc.ScanSource(),
|
||||
timeout: DefaultControlTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ScanSourceClient) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.PollChanges(callCtx, req)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `go test ./internal/pluginhost/ -run TestClientScanSourceRequiresCapability`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/pluginhost/client.go internal/pluginhost/client_test.go
|
||||
git commit -m "feat(pluginhost): scan_source.v1 capability client wrapper"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `plugins.Service` resolver for `scan_source.v1`
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/plugins/service.go`
|
||||
|
||||
- [ ] **Step 1: Add the resolver method**
|
||||
|
||||
After `ScheduledTaskClient` (l.413–423), add the identical shape for scan source:
|
||||
|
||||
```go
|
||||
func (s *Service) ScanSourceClient(
|
||||
ctx context.Context,
|
||||
installationID int,
|
||||
capabilityID string,
|
||||
) (*pluginhost.ScanSourceClient, error) {
|
||||
client, err := s.ensureClient(ctx, installationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.ScanSource(capabilityID)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Build to verify it compiles**
|
||||
|
||||
Run: `go build ./internal/plugins/`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/plugins/service.go
|
||||
git commit -m "feat(plugins): expose scan_source.v1 client resolver"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Schema — `autoscan_connections` + `autoscan_sources`
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/172_autoscan_v2.up.sql`, `migrations/172_autoscan_v2.down.sql`
|
||||
|
||||
- [ ] **Step 1: Write the up migration**
|
||||
|
||||
Create `migrations/172_autoscan_v2.up.sql`. It drops PR #43's `171` tables and creates the decoupled schema (no FK into `request_integrations`; the optional reuse link is `ON DELETE SET NULL`):
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS public.autoscan_sources;
|
||||
DROP TABLE IF EXISTS public.autoscan_settings;
|
||||
|
||||
CREATE TABLE public.autoscan_settings (
|
||||
id boolean PRIMARY KEY DEFAULT true,
|
||||
enabled boolean NOT NULL DEFAULT false,
|
||||
default_poll_interval_seconds integer NOT NULL DEFAULT 600,
|
||||
debounce_seconds integer NOT NULL DEFAULT 60,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT autoscan_settings_singleton CHECK (id),
|
||||
CONSTRAINT autoscan_settings_interval_pos CHECK (default_poll_interval_seconds > 0),
|
||||
CONSTRAINT autoscan_settings_debounce_nonneg CHECK (debounce_seconds >= 0)
|
||||
);
|
||||
INSERT INTO public.autoscan_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
CREATE TABLE public.autoscan_connections (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name text NOT NULL,
|
||||
kind text NOT NULL,
|
||||
-- Own credentials (used when request_integration_id IS NULL):
|
||||
base_url text,
|
||||
api_key_ref text,
|
||||
-- Optional soft reuse of a Requests arr server (live link; SET NULL on delete):
|
||||
request_integration_id text REFERENCES public.request_integrations(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT autoscan_connections_source_present
|
||||
CHECK (request_integration_id IS NOT NULL OR base_url IS NOT NULL)
|
||||
);
|
||||
|
||||
CREATE TABLE public.autoscan_sources (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
installation_id integer NOT NULL,
|
||||
capability_id text NOT NULL,
|
||||
connection_id uuid NOT NULL REFERENCES public.autoscan_connections(id) ON DELETE RESTRICT,
|
||||
enabled boolean NOT NULL DEFAULT false,
|
||||
poll_interval_seconds integer,
|
||||
marker text,
|
||||
last_run_at timestamptz,
|
||||
last_error text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT autoscan_sources_interval_pos CHECK (poll_interval_seconds IS NULL OR poll_interval_seconds > 0),
|
||||
CONSTRAINT autoscan_sources_capability_unique UNIQUE (installation_id, capability_id)
|
||||
);
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write the down migration**
|
||||
|
||||
Create `migrations/172_autoscan_v2.down.sql`:
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS public.autoscan_sources;
|
||||
DROP TABLE IF EXISTS public.autoscan_connections;
|
||||
DROP TABLE IF EXISTS public.autoscan_settings;
|
||||
```
|
||||
|
||||
(Recreating the `171` tables on down-migration is unnecessary: `171` is unreleased and superseded; the down simply removes `172`'s tables.)
|
||||
|
||||
- [ ] **Step 3: Apply and verify against a disposable DB**
|
||||
|
||||
Run the migrations against a throwaway Postgres (the project's disposable-DB pattern) and confirm boot applies cleanly:
|
||||
```bash
|
||||
docker compose up -d postgres
|
||||
go run ./cmd/silo --migrate-only 2>&1 | grep -i "migrations applied" || echo "check migration output"
|
||||
```
|
||||
Expected: migrations apply without error; `\d autoscan_connections` shows the soft FK `request_integration_id`.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add migrations/172_autoscan_v2.up.sql migrations/172_autoscan_v2.down.sql
|
||||
git commit -m "feat(migrations): autoscan v2 schema (connections + sources, no requests FK)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Types and repository
|
||||
|
||||
**Files:**
|
||||
- Rewrite: `internal/autoscan/types.go`
|
||||
- Rewrite: `internal/autoscan/repository.go`
|
||||
|
||||
- [ ] **Step 1: Define the types**
|
||||
|
||||
Replace `internal/autoscan/types.go` with the v2 model:
|
||||
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import "time"
|
||||
|
||||
type Settings struct {
|
||||
Enabled bool
|
||||
DefaultPollIntervalSeconds int
|
||||
DebounceSeconds int
|
||||
}
|
||||
|
||||
// Connection is an arr server the host can reach: either own credentials, or a
|
||||
// live reference to a Requests integration (RequestIntegrationID set).
|
||||
type Connection struct {
|
||||
ID string
|
||||
Name string
|
||||
Kind string
|
||||
BaseURL string // own; empty when linked
|
||||
APIKeyRef string // own; empty when linked
|
||||
RequestIntegrationID *string
|
||||
}
|
||||
|
||||
// Source ties a scan_source plugin capability instance to a connection plus the
|
||||
// host-owned scheduling/bookkeeping state.
|
||||
type Source struct {
|
||||
ID string
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
ConnectionID string
|
||||
Enabled bool
|
||||
PollIntervalSeconds *int // nil => use settings default
|
||||
Marker *string // opaque; nil on first run
|
||||
LastRunAt *time.Time
|
||||
LastError *string
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write a failing repository test**
|
||||
|
||||
Add `internal/autoscan/repository_test.go` exercising round-trip + `AdvanceMarker` against a disposable DB (gate with the project's DB-test build tag/helper if one exists; otherwise a `//go:build integration` tag matching sibling DB tests). Minimal:
|
||||
|
||||
```go
|
||||
func TestRepositoryMarkerRoundTrip(t *testing.T) {
|
||||
repo := newTestRepo(t) // disposable-DB helper used by sibling repo tests
|
||||
connID := repo.mustCreateConnection(t, Connection{Name: "Sonarr", Kind: "sonarr", BaseURL: "http://x", APIKeyRef: "k"})
|
||||
srcID := repo.mustCreateSource(t, Source{InstallationID: 1, CapabilityID: "arr", ConnectionID: connID, Enabled: true})
|
||||
|
||||
if err := repo.AdvanceMarker(ctx, srcID, "2026-06-02T14:10:00Z"); err != nil {
|
||||
t.Fatalf("AdvanceMarker: %v", err)
|
||||
}
|
||||
got, _ := repo.ListEnabledSources(ctx)
|
||||
if len(got) != 1 || got[0].Marker == nil || *got[0].Marker != "2026-06-02T14:10:00Z" {
|
||||
t.Fatalf("marker not persisted: %+v", got)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run to verify it fails**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestRepositoryMarkerRoundTrip` (with the DB helper available)
|
||||
Expected: FAIL — undefined repository methods.
|
||||
|
||||
- [ ] **Step 4: Implement the repository**
|
||||
|
||||
Rewrite `internal/autoscan/repository.go` with: `GetSettings`, `UpdateSettings`, connection CRUD (`CreateConnection`, `UpdateConnection`, `DeleteConnection`, `ListConnections`, `GetConnection`), source CRUD (`UpsertSource`, `ListEnabledSources`, `ListSources`, `GetSource`), and `AdvanceMarker(ctx, sourceID, marker string)` (sets `marker`, `last_run_at = now()`, clears `last_error`). Use pgx; follow the column/scan style of the existing (pre-rewrite) `repository.go` and `internal/requests/repository.go`.
|
||||
|
||||
- [ ] **Step 5: Run to verify it passes**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestRepositoryMarkerRoundTrip`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/types.go internal/autoscan/repository.go internal/autoscan/repository_test.go
|
||||
git commit -m "feat(autoscan): v2 types and repository (connections, sources, markers)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Connection resolution (own or Requests-linked)
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/connection.go`
|
||||
- Create: `internal/autoscan/connection_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```go
|
||||
func TestResolveConnectionOwnVsLinked(t *testing.T) {
|
||||
reqLookup := fakeRequestIntegrationLookup{"req-1": creds{BaseURL: "http://req:7878", APIKeyRef: "rk"}}
|
||||
secrets := fakeSecrets{"rk": "REQKEY", "ownref": "OWNKEY"}
|
||||
r := NewConnectionResolver(reqLookup, secrets)
|
||||
|
||||
own, err := r.Resolve(ctx, Connection{BaseURL: "http://own:8989", APIKeyRef: "ownref"})
|
||||
if err != nil || own.BaseURL != "http://own:8989" || own.APIKey != "OWNKEY" {
|
||||
t.Fatalf("own resolve = %+v, err=%v", own, err)
|
||||
}
|
||||
id := "req-1"
|
||||
linked, err := r.Resolve(ctx, Connection{RequestIntegrationID: &id})
|
||||
if err != nil || linked.BaseURL != "http://req:7878" || linked.APIKey != "REQKEY" {
|
||||
t.Fatalf("linked resolve = %+v, err=%v", linked, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinkedMissingFallsBack(t *testing.T) {
|
||||
r := NewConnectionResolver(fakeRequestIntegrationLookup{}, fakeSecrets{})
|
||||
id := "gone"
|
||||
if _, err := r.Resolve(ctx, Connection{RequestIntegrationID: &id}); err == nil {
|
||||
t.Fatal("expected error when linked Requests integration is missing")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestResolveConnection`
|
||||
Expected: FAIL — undefined `NewConnectionResolver`.
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
Create `internal/autoscan/connection.go`:
|
||||
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ResolvedConnection is concrete credentials handed to the plugin.
|
||||
type ResolvedConnection struct {
|
||||
BaseURL string
|
||||
APIKey string
|
||||
}
|
||||
|
||||
type RequestIntegrationLookup interface {
|
||||
// Returns base URL + encrypted api key ref for a Requests integration.
|
||||
Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error)
|
||||
}
|
||||
|
||||
type SecretResolver interface {
|
||||
Get(ctx context.Context, ref string) (string, error)
|
||||
}
|
||||
|
||||
type ConnectionResolver struct {
|
||||
requests RequestIntegrationLookup
|
||||
secrets SecretResolver
|
||||
}
|
||||
|
||||
func NewConnectionResolver(r RequestIntegrationLookup, s SecretResolver) *ConnectionResolver {
|
||||
return &ConnectionResolver{requests: r, secrets: s}
|
||||
}
|
||||
|
||||
func (cr *ConnectionResolver) Resolve(ctx context.Context, c Connection) (ResolvedConnection, error) {
|
||||
baseURL, apiKeyRef := c.BaseURL, c.APIKeyRef
|
||||
if c.RequestIntegrationID != nil {
|
||||
u, ref, err := cr.requests.Get(ctx, *c.RequestIntegrationID)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: %w", *c.RequestIntegrationID, err)
|
||||
}
|
||||
baseURL, apiKeyRef = u, ref
|
||||
}
|
||||
apiKey := apiKeyRef
|
||||
if cr.secrets != nil && apiKeyRef != "" {
|
||||
resolved, err := cr.secrets.Get(ctx, apiKeyRef)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: resolve api key: %w", err)
|
||||
}
|
||||
if resolved != "" {
|
||||
apiKey = resolved
|
||||
}
|
||||
}
|
||||
return ResolvedConnection{BaseURL: baseURL, APIKey: apiKey}, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestResolveConnection`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/connection.go internal/autoscan/connection_test.go
|
||||
git commit -m "feat(autoscan): resolve connections (own credentials or Requests-linked)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Provider seam over the plugin resolver
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/provider.go`
|
||||
|
||||
- [ ] **Step 1: Define the seam and adapter**
|
||||
|
||||
The engine must be testable without a live plugin, so it depends on a narrow interface, with a production adapter over `plugins.Service.ScanSourceClient`. Create `internal/autoscan/provider.go`:
|
||||
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
)
|
||||
|
||||
// ScanSourceProvider yields changed paths for one source. The engine calls
|
||||
// PollChanges; production wraps the plugins.Service scan_source resolver.
|
||||
type ScanSourceProvider interface {
|
||||
PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection) (paths []string, nextMarker string, err error)
|
||||
}
|
||||
|
||||
// pluginScanSourceClient is the slice of *pluginhost.ScanSourceClient used here.
|
||||
type pluginScanSourceClient interface {
|
||||
PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error)
|
||||
}
|
||||
|
||||
type scanSourceResolver interface {
|
||||
ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (pluginScanSourceClient, error)
|
||||
}
|
||||
|
||||
type pluginProvider struct{ resolver scanSourceResolver }
|
||||
|
||||
func NewPluginProvider(resolver scanSourceResolver) ScanSourceProvider {
|
||||
return &pluginProvider{resolver: resolver}
|
||||
}
|
||||
|
||||
func (p *pluginProvider) PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection) ([]string, string, error) {
|
||||
client, err := p.resolver.ScanSourceClient(ctx, installationID, capabilityID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
resp, err := client.PollChanges(ctx, &pluginv1.PollChangesRequest{CapabilityId: capabilityID, Marker: marker})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return resp.GetChangedPaths(), resp.GetNextMarker(), nil
|
||||
}
|
||||
```
|
||||
|
||||
Note: the connection (`conn`) is delivered to the plugin out-of-band as runtime config when the source is configured (the plugin reads its own `{base_url, api_key}`); it is passed here so a future provider variant could inject per-call. For v1 the production path configures the plugin instance with the resolved connection at upsert time; document this in `NewPluginProvider`'s comment.
|
||||
|
||||
- [ ] **Step 2: Build**
|
||||
|
||||
Run: `go build ./internal/autoscan/`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/provider.go
|
||||
git commit -m "feat(autoscan): scan-source provider seam over the plugin resolver"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Generalize the engine (`Service.PollOnce`)
|
||||
|
||||
**Files:**
|
||||
- Rewrite: `internal/autoscan/service.go`
|
||||
- Modify: `internal/autoscan/service_test.go` (generalize the salvaged tests)
|
||||
|
||||
- [ ] **Step 1: Adapt the salvaged tests to the provider seam**
|
||||
|
||||
The PR #43 `service_test.go` used a `fakeHistory`; replace it with a `fakeProvider` implementing `ScanSourceProvider`, keeping the same assertions (dedupe to one folder; distinct paths same folder → two scans; disabled no-op; provider error keeps marker; enqueue failure releases claims + keeps marker). Add an opaque-marker assertion: the provider's returned `nextMarker` is stored verbatim via `AdvanceMarker`.
|
||||
|
||||
```go
|
||||
type fakeProvider struct {
|
||||
paths map[string][]string // key: capabilityID
|
||||
nextMarker string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeProvider) PollChanges(_ context.Context, _ int, capabilityID, _ string, _ ResolvedConnection) ([]string, string, error) {
|
||||
if f.err != nil {
|
||||
return nil, "", f.err
|
||||
}
|
||||
return f.paths[capabilityID], f.nextMarker, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify the tests fail**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestPollOnce`
|
||||
Expected: FAIL — `Service` signature/`fakeProvider` not yet wired.
|
||||
|
||||
- [ ] **Step 3: Rewrite the engine**
|
||||
|
||||
Rewrite `internal/autoscan/service.go`. Keep the salvaged resolve→suppress→enqueue loop verbatim (it is provider-agnostic already), but drive it from sources + the provider seam:
|
||||
|
||||
```go
|
||||
func (s *Service) PollOnce(ctx context.Context) error {
|
||||
settings, err := s.store.GetSettings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !settings.Enabled {
|
||||
return nil
|
||||
}
|
||||
sources, err := s.store.ListEnabledSources(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ttl := time.Duration(settings.DebounceSeconds) * time.Second
|
||||
|
||||
for _, src := range sources {
|
||||
conn, cerr := s.resolveConnection(ctx, src.ConnectionID)
|
||||
if cerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: resolve connection failed", "source_id", src.ID, "err", cerr)
|
||||
continue
|
||||
}
|
||||
marker := ""
|
||||
if src.Marker != nil {
|
||||
marker = *src.Marker
|
||||
}
|
||||
paths, next, perr := s.provider.PollChanges(ctx, src.InstallationID, src.CapabilityID, marker, conn)
|
||||
if perr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: poll changes failed", "source_id", src.ID, "err", perr)
|
||||
_ = s.store.RecordError(ctx, src.ID, perr.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
targets, claimed := s.resolveAndClaim(ctx, paths, ttl) // salvaged dedupe→resolve→suppress
|
||||
if len(targets) > 0 {
|
||||
if eerr := s.queue.EnqueueScans(ctx, targets); eerr != nil {
|
||||
s.releaseClaims(ctx, claimed)
|
||||
slog.WarnContext(ctx, "autoscan: enqueue failed", "source_id", src.ID, "err", eerr)
|
||||
continue // do NOT advance marker
|
||||
}
|
||||
}
|
||||
if aerr := s.store.AdvanceMarker(ctx, src.ID, next); aerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: advance marker failed", "source_id", src.ID, "err", aerr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
Extract the salvaged loop body into `resolveAndClaim`/`releaseClaims` helpers (lifted verbatim from PR #43's `service.go` lines that build `targets`/`claimed`). Keep `uniqueParentDirs`, `Suppressor`, the `fmt.Sprintf("%d|%s", folderID, target.Path)` suppression key, and the `scantrigger.RequestError` quiet-skip.
|
||||
|
||||
- [ ] **Step 4: Run to verify the tests pass**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestPollOnce`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/service.go internal/autoscan/service_test.go
|
||||
git commit -m "feat(autoscan): generic engine drives sources via scan_source provider"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Delete arr-specific salvage residue
|
||||
|
||||
**Files:**
|
||||
- Delete: `internal/autoscan/history.go`, `history_test.go`, `rewrite.go`, `rewrite_test.go`, `suggest.go`, `suggest_deps.go`, `suggest_test.go`
|
||||
|
||||
- [ ] **Step 1: Remove the files**
|
||||
|
||||
```bash
|
||||
git rm internal/autoscan/history.go internal/autoscan/history_test.go \
|
||||
internal/autoscan/rewrite.go internal/autoscan/rewrite_test.go \
|
||||
internal/autoscan/suggest.go internal/autoscan/suggest_deps.go internal/autoscan/suggest_test.go
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Build to confirm no remaining references**
|
||||
|
||||
Run: `go build ./internal/autoscan/...`
|
||||
Expected: exits 0 (the engine no longer references the deleted arr logic). If it fails, a reference was missed — remove it.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -m "refactor(autoscan): drop arr-specific history/rewrite/suggest (moved to plugin)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Admin API rewrite
|
||||
|
||||
**Files:**
|
||||
- Rewrite: `internal/api/handlers/autoscan.go`
|
||||
- Modify: `internal/api/handlers/autoscan_test.go`
|
||||
|
||||
- [ ] **Step 1: Define the endpoints**
|
||||
|
||||
Rewrite the handler to back the new model. Endpoints (admin-gated, mounted in Task 10):
|
||||
- `GET/PUT /admin/autoscan/settings`
|
||||
- `GET/POST /admin/autoscan/connections`, `PUT/DELETE /admin/autoscan/connections/{id}`
|
||||
- `GET /admin/autoscan/sources`, `PUT /admin/autoscan/sources/{id}` (enable, interval, connection binding)
|
||||
- `POST /admin/autoscan/trigger` (async `PollOnce`, 202 — keep the detached-goroutine + channel-tested pattern from the closed PR's `autoscan_test.go`)
|
||||
- `GET /admin/autoscan/status`
|
||||
|
||||
Strip credentials from responses (never emit `api_key_ref`/resolved keys), matching the `_SENSITIVE_METADATA_KEYS` defense-in-depth posture.
|
||||
|
||||
- [ ] **Step 2: Port + extend the handler tests**
|
||||
|
||||
Carry over the trigger test (the `done`-channel synchronization fix from `d3ee373`), the FK→404 test (now: unknown source/connection → 404), and a secrets-not-leaked test for the connections list.
|
||||
|
||||
- [ ] **Step 3: Run the handler tests (libvips container)**
|
||||
|
||||
Run (in the libvips-equipped container per Prerequisites #3):
|
||||
`go test ./internal/api/handlers/ -run Autoscan`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/api/handlers/autoscan.go internal/api/handlers/autoscan_test.go
|
||||
git commit -m "feat(api): autoscan v2 admin endpoints (settings, connections, sources)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 10: Wiring + rate limits + retire old coupling
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/api/router.go`
|
||||
- Modify: `cmd/silo/main.go`
|
||||
|
||||
- [ ] **Step 1: Rebuild the service wiring**
|
||||
|
||||
In `internal/api/router.go` and `cmd/silo/main.go`, construct the v2 service:
|
||||
|
||||
```go
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB)
|
||||
provider := autoscan.NewPluginProvider(pluginServiceScanSourceAdapter{pluginService})
|
||||
connResolver := autoscan.NewConnectionResolver(requestIntegrationLookup{requestsRepo}, serverSettingsSecretResolver{settingsRepo})
|
||||
autoscanSvc := autoscan.NewService(autoscanRepo, provider, connResolver, deps.Resolver, deps.ScanQueue, autoscan.NewRedisSuppressor(deps.RedisClient))
|
||||
autoscanHandler = handlers.NewAutoscanHandler(autoscanRepo, autoscanSvc)
|
||||
```
|
||||
|
||||
Replace the old route block with the Task 9 routes. Keep the `catalog_*`/admin rate-limit conventions consistent with the security standards (per-user/admin caps on the new endpoints; mirror the request-integrations admin tier).
|
||||
|
||||
- [ ] **Step 2: Retarget the poll task**
|
||||
|
||||
`internal/taskmanager/tasks/autoscan_poll.go` calls `Service.PollOnce` — unchanged behavior; confirm it compiles against the rebuilt service and uses `Settings.DefaultPollIntervalSeconds` for the interval (per-source overrides are honored inside `PollOnce`).
|
||||
|
||||
- [ ] **Step 3: Build the whole tree**
|
||||
|
||||
Run (libvips container): `go build ./...`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/api/router.go cmd/silo/main.go internal/taskmanager/tasks/autoscan_poll.go
|
||||
git commit -m "feat(autoscan): wire v2 service, routes, and poll task"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 11: Full sweep + finalize SDK dependency
|
||||
|
||||
**Files:**
|
||||
- Modify: `go.mod`, `go.sum`
|
||||
|
||||
- [ ] **Step 1: Full build + test (libvips container)**
|
||||
|
||||
Run in the libvips-equipped container (Prerequisites #3):
|
||||
```bash
|
||||
go build ./... && go test ./... && go vet ./...
|
||||
```
|
||||
Expected: build OK; all packages `ok`; vet clean. `gofmt -l internal migrations` prints nothing.
|
||||
|
||||
- [ ] **Step 2: Finalize the SDK version (after `v0.5.0` is tagged upstream)**
|
||||
|
||||
Once `silo-plugin-sdk` PR #2 is merged and `v0.5.0` tagged:
|
||||
```bash
|
||||
go mod edit -dropreplace github.com/Silo-Server/silo-plugin-sdk
|
||||
go get github.com/Silo-Server/silo-plugin-sdk@v0.5.0
|
||||
go mod tidy
|
||||
go build ./...
|
||||
```
|
||||
Expected: builds against the released tag with no replace directive.
|
||||
|
||||
If `v0.5.0` is not yet tagged when this plan is otherwise complete, leave the replace directive in place, commit it, and note in the PR that the dependency must be finalized before merge (do not merge a `replace` pointing at a local path).
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add go.mod go.sum
|
||||
git commit -m "build: depend on silo-plugin-sdk v0.5.0 (scan_source.v1)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** §7 engine (Tasks 6–7), §8 connections incl. reuse-or-own + soft Requests link (Tasks 3,5), §9 data model decoupled from Requests + Autoscan-owned state (Tasks 3–4,9–10), §5/§11 host consumes `scan_source.v1` (Tasks 1–2). §10 (arr plugin) and §9 UI are separate plans.
|
||||
- **Salvage map honored (§13):** engine/dedupe/suppress kept and generalized (Tasks 6–8); arr history/rewrite/suggest deleted (Task 8); `last_poll_at` → opaque `marker` (Tasks 3–4,7).
|
||||
- **Out of scope here:** the Autoscan admin **UI** category (Part 2 plan) and the **arr plugin** (separate repo plan). This backend is testable on its own via the repository/engine/handler tests.
|
||||
- **Risk carried from spec §14:** plugin multiplicity (one installation per arr server vs. many connections per installation) is modeled as one `autoscan_sources` row per `(installation_id, capability_id)`; if the runtime supports multiple capability instances per installation this maps cleanly, otherwise it implies one installation per server. Confirm during Task 1–2 by inspecting `ensureClient`/installation handling.
|
||||
- **Type consistency:** `ResolvedConnection{BaseURL,APIKey}`, `Connection{BaseURL,APIKeyRef,RequestIntegrationID}`, `Source{InstallationID,CapabilityID,ConnectionID,Marker}`, `ScanSourceProvider.PollChanges(...)`, `Service.PollOnce` used consistently across Tasks 4–10.
|
||||
@@ -0,0 +1,694 @@
|
||||
# Autoscan Rewrite-Sync Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** A "Sync rewrites from arr" action that suggests autoscan `path_rewrites` for an instance by matching its arr root folders to Silo media folders on shared trailing path segments, previewed and committed by the admin.
|
||||
|
||||
**Architecture:** A pure `suggestRewrites` matcher in `internal/autoscan`, a `Service.SuggestRewrites` method backed by two new injected deps (an arr root-folder client and a Silo folder lister), one read-only admin endpoint returning a preview, and a "Sync from arr" button in the existing Autoscan tab that merges proposals into the editor (admin saves via the existing per-source PUT). Manual rewrites stay the source of truth.
|
||||
|
||||
**Tech Stack:** Go (standard `testing`), React/TypeScript.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-06-02-autoscan-rewrite-sync-design.md`
|
||||
|
||||
**Commands assume the repository root is the cwd.** Go tests: `go test ./internal/autoscan/...`. Frontend: `cd web && pnpm exec tsc -b && pnpm run lint`. Ensure the Go toolchain is on `PATH` (prepend its `bin` directory if `go` is missing). `internal/api`/`internal/api/handlers` cannot be compiled without libvips; verify those via `gofmt` + the Docker build.
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 — Branch
|
||||
|
||||
- [ ] **Step 0.1: Confirm branch**
|
||||
|
||||
```bash
|
||||
git rev-parse --abbrev-ref HEAD # expect: feat/autoscan-arr-polling
|
||||
```
|
||||
This feature extends the autoscan branch. If you're elsewhere: `git checkout feat/autoscan-arr-polling`.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Core matcher (pure, TDD)
|
||||
|
||||
### Task 1: suggestRewrites + types
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/suggest.go`
|
||||
- Create: `internal/autoscan/suggest_test.go`
|
||||
|
||||
- [ ] **Step 1.1: Write the failing test**
|
||||
|
||||
`internal/autoscan/suggest_test.go`:
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSuggestRewrites(t *testing.T) {
|
||||
silo := []string{
|
||||
"/mnt/media/happy/storage2/tvshows1",
|
||||
"/mnt/media/happy4k/4ktv7",
|
||||
"/mnt/media/storage/Anime/Subs",
|
||||
"/mnt/media/storage/Anime2/Subs",
|
||||
"/library/Films",
|
||||
"/tank/television/Show",
|
||||
}
|
||||
|
||||
t.Run("multi-segment unique match", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/mnt/happy/storage2/tvshows1"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" || got.Proposed[0].MatchDepth != 2 {
|
||||
t.Fatalf("proposed=%+v", got.Proposed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("single-segment unique match (different parents)", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/mnt/kodama/storage2/4ktv7"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy4k/4ktv7" || got.Proposed[0].MatchDepth != 1 {
|
||||
t.Fatalf("proposed=%+v", got.Proposed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("longest-suffix disambiguation", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/mnt/kodama/storage1/Anime/Subs"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/storage/Anime/Subs" || got.Proposed[0].MatchDepth != 2 {
|
||||
t.Fatalf("expected Anime/Subs (depth 2), got %+v", got.Proposed)
|
||||
}
|
||||
if len(got.Ambiguous) != 0 {
|
||||
t.Fatalf("should not be ambiguous: %+v", got.Ambiguous)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unmatched when no shared segment", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/data/Movies"}, silo, nil)
|
||||
if len(got.Unmatched) != 1 || got.Unmatched[0] != "/data/Movies" || len(got.Proposed) != 0 {
|
||||
t.Fatalf("got=%+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("leaf match across unlike layouts", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/srv/tv/Show"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/tank/television/Show" {
|
||||
t.Fatalf("got=%+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ambiguous tie", func(t *testing.T) {
|
||||
// a root whose only shared segment is "Subs" ties two folders
|
||||
got := suggestRewrites([]string{"/foo/bar/Subs"}, silo, nil)
|
||||
if len(got.Ambiguous) != 1 || len(got.Ambiguous[0].Candidates) != 2 {
|
||||
t.Fatalf("expected ambiguous with 2 candidates, got %+v", got.Ambiguous)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("covered by existing rule", func(t *testing.T) {
|
||||
existing := []PathRewrite{{From: "/mnt/happy", To: "/mnt/media/happy"}}
|
||||
got := suggestRewrites([]string{"/mnt/happy/storage2/tvshows1"}, silo, existing)
|
||||
if len(got.Covered) != 1 || len(got.Proposed) != 0 {
|
||||
t.Fatalf("expected covered, got %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normalization: trailing slash, backslashes, dup slashes", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{`\mnt\happy\\storage2\tvshows1\`}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].From != "/mnt/happy/storage2/tvshows1" || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" {
|
||||
t.Fatalf("normalization failed: %+v", got.Proposed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCommonSuffixLen(t *testing.T) {
|
||||
cases := []struct {
|
||||
a, b []string
|
||||
want int
|
||||
}{
|
||||
{[]string{"a", "b", "c"}, []string{"x", "b", "c"}, 2},
|
||||
{[]string{"4ktv7"}, []string{"happy4k", "4ktv7"}, 1},
|
||||
{[]string{"4ktv7"}, []string{"4ktv70"}, 0}, // full-segment, not substring
|
||||
{[]string{"a"}, []string{"b"}, 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := commonSuffixLen(tc.a, tc.b); got != tc.want {
|
||||
t.Fatalf("commonSuffixLen(%v,%v)=%d want %d", tc.a, tc.b, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 1.2: Run it — expect failure**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/ -run 'TestSuggestRewrites|TestCommonSuffixLen' -v
|
||||
```
|
||||
Expected: FAIL (`suggestRewrites` undefined).
|
||||
|
||||
- [ ] **Step 1.3: Implement** `internal/autoscan/suggest.go`:
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import "strings"
|
||||
|
||||
// RewriteSuggestions is the result of matching arr root folders to Silo folders.
|
||||
type RewriteSuggestions struct {
|
||||
Proposed []ProposedRewrite `json:"proposed"`
|
||||
Unmatched []string `json:"unmatched"`
|
||||
Ambiguous []AmbiguousRoot `json:"ambiguous"`
|
||||
Covered []string `json:"covered"`
|
||||
}
|
||||
|
||||
// ProposedRewrite is a suggested rewrite plus its confidence (shared trailing segments).
|
||||
type ProposedRewrite struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
MatchDepth int `json:"match_depth"`
|
||||
}
|
||||
|
||||
// AmbiguousRoot is an arr root that tied across multiple Silo folders.
|
||||
type AmbiguousRoot struct {
|
||||
Root string `json:"root"`
|
||||
Candidates []string `json:"candidates"`
|
||||
}
|
||||
|
||||
// normalizePath makes a path comparable: backslashes -> '/', collapse duplicate
|
||||
// slashes, strip a trailing slash (but keep a bare "/").
|
||||
func normalizePath(p string) string {
|
||||
p = normalizeSeparators(strings.TrimSpace(p))
|
||||
for strings.Contains(p, "//") {
|
||||
p = strings.ReplaceAll(p, "//", "/")
|
||||
}
|
||||
if len(p) > 1 {
|
||||
p = strings.TrimRight(p, "/")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func segments(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
// commonSuffixLen counts equal trailing segments (full-segment equality).
|
||||
func commonSuffixLen(a, b []string) int {
|
||||
i, j, n := len(a)-1, len(b)-1, 0
|
||||
for i >= 0 && j >= 0 && a[i] == b[j] {
|
||||
n++
|
||||
i--
|
||||
j--
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// coveredBy reports whether an existing rewrite already matches root (same
|
||||
// boundary rule as applyRewrites).
|
||||
func coveredBy(root string, existing []PathRewrite) bool {
|
||||
for _, rw := range existing {
|
||||
from := strings.TrimRight(strings.TrimSpace(rw.From), "/")
|
||||
if from == "" {
|
||||
continue
|
||||
}
|
||||
if root == from || strings.HasPrefix(root, from+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// suggestRewrites matches each arr root to the Silo folder sharing the most
|
||||
// trailing path segments (unique winner). Roots already handled by an existing
|
||||
// rewrite are reported as Covered; roots with no shared segment as Unmatched;
|
||||
// ties as Ambiguous. Pure: no I/O, no deployment constants.
|
||||
func suggestRewrites(arrRoots, siloFolderPaths []string, existing []PathRewrite) RewriteSuggestions {
|
||||
siloNorm := make([]string, 0, len(siloFolderPaths))
|
||||
siloSegs := make([][]string, 0, len(siloFolderPaths))
|
||||
for _, p := range siloFolderPaths {
|
||||
n := normalizePath(p)
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
siloNorm = append(siloNorm, n)
|
||||
siloSegs = append(siloSegs, segments(n))
|
||||
}
|
||||
|
||||
var out RewriteSuggestions
|
||||
for _, raw := range arrRoots {
|
||||
root := normalizePath(raw)
|
||||
if root == "" {
|
||||
continue
|
||||
}
|
||||
if coveredBy(root, existing) {
|
||||
out.Covered = append(out.Covered, root)
|
||||
continue
|
||||
}
|
||||
rootSegs := segments(root)
|
||||
best := 0
|
||||
var winners []string
|
||||
for i, segs := range siloSegs {
|
||||
n := commonSuffixLen(rootSegs, segs)
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
if n > best {
|
||||
best, winners = n, []string{siloNorm[i]}
|
||||
} else if n == best {
|
||||
winners = append(winners, siloNorm[i])
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case best == 0:
|
||||
out.Unmatched = append(out.Unmatched, root)
|
||||
case len(winners) == 1:
|
||||
out.Proposed = append(out.Proposed, ProposedRewrite{From: root, To: winners[0], MatchDepth: best})
|
||||
default:
|
||||
out.Ambiguous = append(out.Ambiguous, AmbiguousRoot{Root: root, Candidates: winners})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 1.4: Run it — expect pass; commit**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/ -run 'TestSuggestRewrites|TestCommonSuffixLen' -v
|
||||
gofmt -l internal/autoscan/suggest.go internal/autoscan/suggest_test.go
|
||||
git add internal/autoscan/suggest.go internal/autoscan/suggest_test.go
|
||||
git commit -m "feat(autoscan): suffix-match rewrite suggester"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Service support
|
||||
|
||||
### Task 2: Repository.GetSource
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/autoscan/repository.go`
|
||||
|
||||
- [ ] **Step 2.1: Implement** `GetSource` (single-source join, includes disabled). Add after `ListEnabledSources`:
|
||||
```go
|
||||
// GetSource returns one instance's autoscan state joined with its
|
||||
// request_integrations row (kind/base_url/api_key_ref), regardless of enabled.
|
||||
func (r *Repository) GetSource(ctx context.Context, integrationID string) (*Source, error) {
|
||||
row := r.pool.QueryRow(ctx, sourceSelect+` WHERE ri.id = $1`, integrationID)
|
||||
src, err := scanSource(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrIntegrationNotFound, integrationID)
|
||||
}
|
||||
return nil, fmt.Errorf("get autoscan source: %w", err)
|
||||
}
|
||||
return &src, nil
|
||||
}
|
||||
```
|
||||
(`errors`, `pgx`, `fmt`, `ErrIntegrationNotFound`, and `sourceSelect`/`scanSource` already exist in the file.)
|
||||
|
||||
- [ ] **Step 2.2: Build + commit**
|
||||
|
||||
```bash
|
||||
go build ./internal/autoscan/... && go test ./internal/autoscan/...
|
||||
git add internal/autoscan/repository.go
|
||||
git commit -m "feat(autoscan): GetSource single-source lookup"
|
||||
```
|
||||
|
||||
### Task 3: RootFolderClient + FolderLister adapters
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/suggest_deps.go`
|
||||
|
||||
- [ ] **Step 3.1: Implement the two adapters + interfaces**
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/requests/arrclient"
|
||||
)
|
||||
|
||||
// RootFolderClient lists a Radarr/Sonarr instance's configured root folder paths.
|
||||
type RootFolderClient interface {
|
||||
RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error)
|
||||
}
|
||||
|
||||
// FolderLister lists every Silo media-folder path.
|
||||
type FolderLister interface {
|
||||
ListFolderPaths(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
type arrRootFolderClient struct{ httpClient *http.Client }
|
||||
|
||||
// NewArrRootFolderClient returns a RootFolderClient backed by the shared arrclient.
|
||||
func NewArrRootFolderClient(httpClient *http.Client) RootFolderClient {
|
||||
return &arrRootFolderClient{httpClient: httpClient}
|
||||
}
|
||||
|
||||
func (c *arrRootFolderClient) RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error) {
|
||||
client := arrclient.New(baseURL, apiKey, c.httpClient)
|
||||
folders, err := arrclient.ListRootFolders(ctx, client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := make([]string, 0, len(folders))
|
||||
for _, f := range folders {
|
||||
if f.Path != "" {
|
||||
paths = append(paths, f.Path)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
type catalogFolderLister struct{ repo *catalog.FolderRepository }
|
||||
|
||||
// NewCatalogFolderLister adapts catalog.FolderRepository to FolderLister.
|
||||
func NewCatalogFolderLister(repo *catalog.FolderRepository) FolderLister {
|
||||
return &catalogFolderLister{repo: repo}
|
||||
}
|
||||
|
||||
func (l *catalogFolderLister) ListFolderPaths(ctx context.Context) ([]string, error) {
|
||||
folders, err := l.repo.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var paths []string
|
||||
for _, f := range folders {
|
||||
paths = append(paths, f.Paths...)
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
```
|
||||
Note: `autoscan` already depends on `catalog` (via `scantrigger`) and on `requests/arrclient` (via `history.go`), so these imports add no new module-level coupling and no cycle (`catalog` does not import `autoscan`).
|
||||
|
||||
- [ ] **Step 3.2: Build + commit**
|
||||
|
||||
```bash
|
||||
go build ./internal/autoscan/...
|
||||
git add internal/autoscan/suggest_deps.go
|
||||
git commit -m "feat(autoscan): arr root-folder client + Silo folder lister"
|
||||
```
|
||||
|
||||
### Task 4: Service.SuggestRewrites
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/autoscan/service.go`
|
||||
- Modify: `internal/autoscan/service_test.go`
|
||||
|
||||
- [ ] **Step 4.1: Write the failing test** (append to `service_test.go`):
|
||||
```go
|
||||
type fakeRootFolders struct {
|
||||
paths []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeRootFolders) RootFolders(context.Context, string, string) ([]string, error) {
|
||||
return f.paths, f.err
|
||||
}
|
||||
|
||||
type fakeFolderLister struct{ paths []string }
|
||||
|
||||
func (f fakeFolderLister) ListFolderPaths(context.Context) ([]string, error) { return f.paths, nil }
|
||||
|
||||
type sourceGetterStore struct {
|
||||
fakeStore
|
||||
src *Source
|
||||
}
|
||||
|
||||
func (s *sourceGetterStore) GetSource(context.Context, string) (*Source, error) { return s.src, nil }
|
||||
|
||||
func TestSuggestRewritesService(t *testing.T) {
|
||||
store := &sourceGetterStore{src: &Source{IntegrationID: "i1", Kind: "sonarr", BaseURL: "http://x", APIKeyRef: "k"}}
|
||||
svc := NewService(store, &fakeHistory{}, fakeResolver{}, &recordingQueuer{}, allowSuppressor{}, nil)
|
||||
svc.SetRewriteResolvers(
|
||||
fakeRootFolders{paths: []string{"/mnt/happy/storage2/tvshows1", "/data/Movies"}},
|
||||
fakeFolderLister{paths: []string{"/mnt/media/happy/storage2/tvshows1"}},
|
||||
)
|
||||
got, err := svc.SuggestRewrites(context.Background(), "i1")
|
||||
if err != nil {
|
||||
t.Fatalf("SuggestRewrites: %v", err)
|
||||
}
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" {
|
||||
t.Fatalf("proposed=%+v", got.Proposed)
|
||||
}
|
||||
if len(got.Unmatched) != 1 || got.Unmatched[0] != "/data/Movies" {
|
||||
t.Fatalf("unmatched=%+v", got.Unmatched)
|
||||
}
|
||||
}
|
||||
```
|
||||
> The `Store` interface needs a `GetSource` method so the service can call it. Add `GetSource(ctx context.Context, integrationID string) (*Source, error)` to the `Store` interface in `service.go`. `*Repository` already implements it (Task 2). The test's `sourceGetterStore` embeds the existing `fakeStore` and adds `GetSource`. If `fakeStore` does not satisfy the widened interface on its own, that's fine — only `sourceGetterStore` is used here; but ensure the package still compiles (other tests construct `fakeStore` directly and pass it where `Store` is required — add a `GetSource` method to `fakeStore` returning `(nil, nil)` so it still satisfies `Store`).
|
||||
|
||||
- [ ] **Step 4.2: Run it — expect failure**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/ -run TestSuggestRewritesService -v
|
||||
```
|
||||
Expected: FAIL (`SetRewriteResolvers`/`SuggestRewrites` undefined).
|
||||
|
||||
- [ ] **Step 4.3: Implement** in `service.go`. Add fields + setter + method, and add `GetSource` to `Store`:
|
||||
```go
|
||||
// add to the Store interface:
|
||||
// GetSource(ctx context.Context, integrationID string) (*Source, error)
|
||||
|
||||
// add fields to Service struct:
|
||||
// rootFolders RootFolderClient
|
||||
// folders FolderLister
|
||||
|
||||
// SetRewriteResolvers wires the deps used by SuggestRewrites (optional; only the
|
||||
// admin-facing service needs them).
|
||||
func (s *Service) SetRewriteResolvers(rootFolders RootFolderClient, folders FolderLister) {
|
||||
s.rootFolders = rootFolders
|
||||
s.folders = folders
|
||||
}
|
||||
|
||||
// SuggestRewrites matches an instance's arr root folders to Silo media folders.
|
||||
func (s *Service) SuggestRewrites(ctx context.Context, integrationID string) (RewriteSuggestions, error) {
|
||||
if s.rootFolders == nil || s.folders == nil {
|
||||
return RewriteSuggestions{}, fmt.Errorf("autoscan: rewrite suggestion not configured")
|
||||
}
|
||||
src, err := s.store.GetSource(ctx, integrationID)
|
||||
if err != nil {
|
||||
return RewriteSuggestions{}, err
|
||||
}
|
||||
apiKey := src.APIKeyRef
|
||||
if s.secrets != nil && apiKey != "" {
|
||||
if resolved, rerr := s.secrets.Get(ctx, apiKey); rerr == nil && resolved != "" {
|
||||
apiKey = resolved
|
||||
}
|
||||
}
|
||||
arrRoots, err := s.rootFolders.RootFolders(ctx, src.BaseURL, apiKey)
|
||||
if err != nil {
|
||||
return RewriteSuggestions{}, fmt.Errorf("autoscan: list arr root folders: %w", err)
|
||||
}
|
||||
siloPaths, err := s.folders.ListFolderPaths(ctx)
|
||||
if err != nil {
|
||||
return RewriteSuggestions{}, fmt.Errorf("autoscan: list silo folders: %w", err)
|
||||
}
|
||||
return suggestRewrites(arrRoots, siloPaths, src.PathRewrites), nil
|
||||
}
|
||||
```
|
||||
Add `GetSource` to `fakeStore` in `service_test.go` (`func (f *fakeStore) GetSource(context.Context, string) (*Source, error) { return nil, nil }`).
|
||||
|
||||
- [ ] **Step 4.4: Run tests — expect pass; commit**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/...
|
||||
gofmt -l internal/autoscan/service.go internal/autoscan/service_test.go
|
||||
git add internal/autoscan/service.go internal/autoscan/service_test.go
|
||||
git commit -m "feat(autoscan): Service.SuggestRewrites"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — API endpoint
|
||||
|
||||
### Task 5: Handler + route
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/api/handlers/autoscan.go`
|
||||
- Modify: `internal/api/router.go`
|
||||
- Modify: `internal/api/handlers/autoscan_test.go`
|
||||
|
||||
- [ ] **Step 5.1: Extend the handler's service interface + add the handler method**
|
||||
|
||||
In `autoscan.go`, add `SuggestRewrites` to the `autoscanTriggerer` interface:
|
||||
```go
|
||||
type autoscanTriggerer interface {
|
||||
PollOnce(ctx context.Context) error
|
||||
SuggestRewrites(ctx context.Context, integrationID string) (autoscan.RewriteSuggestions, error)
|
||||
}
|
||||
```
|
||||
Add the handler:
|
||||
```go
|
||||
func (h *AutoscanHandler) HandleRewriteSuggestions(w http.ResponseWriter, r *http.Request) {
|
||||
id := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
suggestions, err := h.svc.SuggestRewrites(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, autoscan.ErrIntegrationNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Autoscan source not found")
|
||||
return
|
||||
}
|
||||
// arr unreachable / bad key / folder list failure
|
||||
writeError(w, http.StatusBadGateway, "arr_unreachable", "Could not load root folders from the arr instance")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, suggestions)
|
||||
}
|
||||
```
|
||||
(`errors`, `autoscan`, `chi`, `strings`, `writeError`, `writeJSON` already imported.)
|
||||
|
||||
- [ ] **Step 5.2: Mount the route** in `router.go`, inside the autoscan route block:
|
||||
```go
|
||||
r.Get("/autoscan/sources/{id}/rewrite-suggestions", autoscanHandler.HandleRewriteSuggestions)
|
||||
```
|
||||
|
||||
- [ ] **Step 5.3: Wire the service deps** in `router.go` where `autoscanSvc` is built (after `autoscan.NewService(...)`):
|
||||
```go
|
||||
autoscanSvc.SetRewriteResolvers(
|
||||
autoscan.NewArrRootFolderClient(nil),
|
||||
autoscan.NewCatalogFolderLister(deps.FolderRepo),
|
||||
)
|
||||
```
|
||||
(This is inside the existing `if deps.FolderRepo != nil && deps.LibraryScanQueue != nil` block, so `deps.FolderRepo` is non-nil here.)
|
||||
|
||||
- [ ] **Step 5.4: Update the handler test fake**
|
||||
|
||||
In `autoscan_test.go`, the fake satisfying `autoscanTriggerer` (e.g. `fakeAutoscanTriggerer`) gains:
|
||||
```go
|
||||
func (f *fakeAutoscanTriggerer) SuggestRewrites(context.Context, string) (autoscan.RewriteSuggestions, error) {
|
||||
return autoscan.RewriteSuggestions{}, nil
|
||||
}
|
||||
```
|
||||
Add a test asserting `HandleRewriteSuggestions` returns 200 with JSON for a known id (mirror the existing handler tests).
|
||||
|
||||
- [ ] **Step 5.5: Verify (limited) + commit**
|
||||
|
||||
```bash
|
||||
# ensure the Go toolchain is on PATH
|
||||
go vet ./internal/autoscan/...
|
||||
gofmt -l internal/api/handlers/autoscan.go internal/api/router.go internal/api/handlers/autoscan_test.go
|
||||
# internal/api cannot compile here (libvips) — Docker verifies later.
|
||||
git add internal/api/handlers/autoscan.go internal/api/router.go internal/api/handlers/autoscan_test.go
|
||||
git commit -m "feat(autoscan): rewrite-suggestions endpoint"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Frontend
|
||||
|
||||
### Task 6: Types + hook
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/api/types.ts`
|
||||
- Modify: `web/src/hooks/queries/useAutoscan.ts`
|
||||
|
||||
- [ ] **Step 6.1: Types** — add to `types.ts`:
|
||||
```ts
|
||||
export interface AutoscanProposedRewrite {
|
||||
from: string;
|
||||
to: string;
|
||||
match_depth: number;
|
||||
}
|
||||
export interface AutoscanAmbiguousRoot {
|
||||
root: string;
|
||||
candidates: string[];
|
||||
}
|
||||
export interface AutoscanRewriteSuggestions {
|
||||
proposed: AutoscanProposedRewrite[];
|
||||
unmatched: string[];
|
||||
ambiguous: AutoscanAmbiguousRoot[];
|
||||
covered: string[];
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6.2: Hook** — add to `useAutoscan.ts` a mutation that fetches suggestions for an id:
|
||||
```ts
|
||||
export function useAutoscanRewriteSuggestions() {
|
||||
return useMutation({
|
||||
mutationFn: (id: string) =>
|
||||
api<AutoscanRewriteSuggestions>(`/admin/autoscan/sources/${id}/rewrite-suggestions`),
|
||||
onError: (err) =>
|
||||
toast.error(err instanceof Error ? err.message : "Could not sync rewrites from the arr instance"),
|
||||
});
|
||||
}
|
||||
```
|
||||
Match the import/`api`/`toast`/`useMutation` conventions already in the file (it imports `api` and `toast` already; reuse them; import the new type).
|
||||
|
||||
- [ ] **Step 6.3: Lint + commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b && pnpm run lint && pnpm exec prettier --check src/api/types.ts src/hooks/queries/useAutoscan.ts
|
||||
git add web/src/api/types.ts web/src/hooks/queries/useAutoscan.ts
|
||||
git commit -m "feat(web): autoscan rewrite-suggestions types and hook"
|
||||
```
|
||||
|
||||
### Task 7: Sync button + preview in AutoscanSourceEditor
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/pages/AdminRequests.tsx`
|
||||
|
||||
- [ ] **Step 7.1: Add the button + preview**
|
||||
|
||||
In `AutoscanSourceEditor`:
|
||||
- Call `const suggest = useAutoscanRewriteSuggestions();` and add local state `const [preview, setPreview] = useState<AutoscanRewriteSuggestions | null>(null);` and `const [selected, setSelected] = useState<Set<string>>(new Set());`.
|
||||
- Add a **"Sync from arr"** `Button` beside Save: `onClick={async () => { const s = await suggest.mutateAsync(source.integration_id); setPreview(s); setSelected(new Set(s.proposed.map((p) => p.from))); }}` (disabled while `suggest.isPending`).
|
||||
- When `preview` is set, render a panel (reuse the file's `Dialog` or an inline bordered panel) with three sections:
|
||||
- **Proposed:** each `proposed[]` row — a checkbox bound to `selected.has(p.from)` (toggle updates the set), `from → to`, and a `Badge` showing confidence: `match_depth >= 2 ? "${match_depth} segments" : "1 segment — weak"` (destructive/secondary variant for weak).
|
||||
- **Unmatched:** muted list of `unmatched[]` ("no Silo match — add manually if needed").
|
||||
- **Ambiguous:** list of `ambiguous[]` (`root` + its `candidates` joined) — info only.
|
||||
- An **"Add selected to rewrites"** button that merges the checked proposals into the editor's existing rewrites state, deduped by `from` (skip any whose `from` already exists), then `setPreview(null)`. Do NOT auto-save — the admin then clicks the existing **Save**.
|
||||
- Keep the existing manual path-rewrite editor unchanged; sync only appends rows to it.
|
||||
|
||||
Reuse existing `Button`/`Badge`/`Field`/`Dialog` primitives and the editor's existing rewrites-state setter.
|
||||
|
||||
- [ ] **Step 7.2: Lint + commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b && pnpm run lint && pnpm exec prettier --check src/pages/AdminRequests.tsx
|
||||
git add web/src/pages/AdminRequests.tsx
|
||||
git commit -m "feat(web): autoscan sync-rewrites preview"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 — Verification
|
||||
|
||||
### Task 8: Full verification
|
||||
|
||||
- [ ] **Step 8.1: Go**
|
||||
|
||||
```bash
|
||||
# ensure the Go toolchain is on PATH
|
||||
go test ./internal/autoscan/... 2>&1 | tail
|
||||
go vet ./internal/autoscan/...
|
||||
gofmt -l internal/autoscan internal/api/handlers/autoscan.go internal/api/router.go
|
||||
```
|
||||
Expected: all pass; vet/gofmt clean.
|
||||
|
||||
- [ ] **Step 8.2: Frontend**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b && pnpm run lint && pnpm run format:check
|
||||
```
|
||||
|
||||
- [ ] **Step 8.3: Docker build (closes the libvips gap)**
|
||||
|
||||
```bash
|
||||
docker build --build-arg BUILD_REVISION=$(git rev-parse HEAD) --build-arg BUILD_DIRTY=false -t silo-server:autoscan .
|
||||
```
|
||||
Expected: exit 0 (handlers + router + frontend compile).
|
||||
|
||||
- [ ] **Step 8.4: Smoke (optional)**
|
||||
|
||||
On a deploy with arr instances configured: open an instance in the Autoscan tab, click **Sync from arr**, confirm the preview shows proposed (with confidence) + unmatched + ambiguous, "Add selected" appends deduped rows, and Save persists them.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes (resolved)
|
||||
|
||||
- **Spec §1 matcher** → Task 1 (incl. normalization, covered-first, suffix depth, unlike-deployment fixtures). **§2 endpoint/service** → Tasks 2–5 (`GetSource`, `RootFolderClient`/`FolderLister`, `SuggestRewrites`, endpoint, 404/502 mapping). **§3 frontend** → Tasks 6–7. **§4 testing** → Tasks 1,4,5 + Task 8.
|
||||
- **Manual-preserving**: sync only *appends* deduped rows to the editor; the admin saves via the existing `PUT` (Task 7). Never auto-writes.
|
||||
- **Generic**: `suggestRewrites` takes plain string slices, no deployment constants; tests include `/data/Movies` and `/srv/tv` fixtures (Task 1).
|
||||
- **Optional deps**: `SetRewriteResolvers` keeps `main.go`'s poll service untouched; `SuggestRewrites` errors if unset (Task 4).
|
||||
- **Type consistency**: `RewriteSuggestions`/`ProposedRewrite`/`AmbiguousRoot` (Go) ↔ `AutoscanRewriteSuggestions`/`AutoscanProposedRewrite`/`AutoscanAmbiguousRoot` (TS); `match_depth` JSON tag matches.
|
||||
@@ -0,0 +1,375 @@
|
||||
# `scan_source.v1` SDK Capability — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a new, additive `scan_source.v1` plugin capability to `silo-plugin-sdk` so the Silo host can pull changed paths from a scan-source plugin on a timer.
|
||||
|
||||
**Architecture:** Mirror the existing `scheduled_task.v1` capability exactly — a single-RPC, request/response gRPC service defined in a `.proto`, generated into `pkg/pluginproto`, then registered in three Go helper sites (`capability.KnownTypes`, the `runtime.CapabilityServers` struct, and the `runtime.GRPCPlugin` server registration). Nothing existing is modified in shape; only new symbols are added. Tag a new SDK release so the host and the arr plugin can build against it.
|
||||
|
||||
**Tech Stack:** Go, Protocol Buffers (proto3), `buf` codegen (`buf.gen.yaml`), `google.golang.org/protobuf`, `google.golang.org/grpc`, the `hashicorp/go-plugin` runtime the SDK wraps.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites (read before starting)
|
||||
|
||||
This plan targets the **`silo-plugin-sdk`** repository (Go module `github.com/Silo-Server/silo-plugin-sdk`), which is **not currently checked out** in this workspace — only `silo-server` and `silo-plugin-tmdb` are. Before executing:
|
||||
|
||||
1. Clone `silo-plugin-sdk` as a writable working copy beside the other repos. Commands below assume **the silo-plugin-sdk repository root is the cwd**.
|
||||
2. Ensure the proto toolchain is available: the `make proto` target auto-installs `buf`, `protoc-gen-go@v1.36.11`, and `protoc-gen-go-grpc@v1.6.1` into `$(GO_BIN)`, but it requires the **`protoc`** binary to already be on `PATH` (it errors `protoc is required` otherwise). Install `protoc` if absent.
|
||||
3. This SDK is the **long pole**: the silo-server host plan and the arr-plugin plan both depend on the version tagged at the end of this plan (Task 6). Do not start those until this is tagged.
|
||||
|
||||
**Pattern reference (current `scheduled_task.v1`, the template this mirrors):**
|
||||
- Proto: `proto/silo/plugin/v1/scheduled_task.proto`
|
||||
- Generated: `pkg/pluginproto/silo/plugin/v1/scheduled_task.pb.go` + `scheduled_task_grpc.pb.go`
|
||||
- Capability const + allowlist: `pkg/pluginsdk/capability/capability.go`
|
||||
- Runtime wiring: `pkg/pluginsdk/runtime/runtime.go` (struct field, `Client` accessor, `GRPCServer` registration)
|
||||
- Manifest acceptance test pattern: `pkg/pluginsdk/manifest/manifest_test.go`
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `proto/silo/plugin/v1/scan_source.proto` | The `ScanSource` service + `PollChanges` request/response messages | Create |
|
||||
| `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go` | Generated message types | Create (codegen) |
|
||||
| `pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go` | Generated client/server stubs | Create (codegen) |
|
||||
| `pkg/pluginsdk/capability/capability.go` | `ScanSource` const + add to `KnownTypes` allowlist | Modify |
|
||||
| `pkg/pluginsdk/capability/capability_test.go` | Assert `ScanSource` is a known type | Create or Modify |
|
||||
| `pkg/pluginsdk/runtime/runtime.go` | `CapabilityServers.ScanSource` field, `Client.ScanSource()` accessor, `GRPCServer` registration | Modify |
|
||||
| `pkg/pluginsdk/runtime/runtime_test.go` | Assert a `ScanSource` server registers without error | Create or Modify |
|
||||
| `pkg/pluginsdk/manifest/manifest_test.go` | Assert a manifest declaring `scan_source.v1` loads | Modify |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Define and generate the `scan_source.v1` proto
|
||||
|
||||
**Files:**
|
||||
- Create: `proto/silo/plugin/v1/scan_source.proto`
|
||||
- Create (via codegen): `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go`, `pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go`
|
||||
|
||||
- [ ] **Step 1: Write the proto**
|
||||
|
||||
Create `proto/silo/plugin/v1/scan_source.proto` (the `go_package` option must match the sibling protos exactly so generated code lands in package `pluginv1`):
|
||||
|
||||
```proto
|
||||
syntax = "proto3";
|
||||
|
||||
package silo.plugin.v1;
|
||||
|
||||
option go_package = "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1;pluginv1";
|
||||
|
||||
// ScanSource lets the host pull changed filesystem paths from a provider
|
||||
// (e.g. Sonarr/Radarr, inotify, Ceph). Pull only: the host owns the timer and
|
||||
// calls PollChanges; the plugin never initiates.
|
||||
service ScanSource {
|
||||
rpc PollChanges(PollChangesRequest) returns (PollChangesResponse);
|
||||
}
|
||||
|
||||
message PollChangesRequest {
|
||||
// Which configured scan_source capability instance is being polled.
|
||||
string capability_id = 1;
|
||||
// Opaque continuation token from the previous PollChanges. Empty on first
|
||||
// run, which the provider treats as "start from now" (do not replay history).
|
||||
string marker = 2;
|
||||
}
|
||||
|
||||
message PollChangesResponse {
|
||||
// Absolute paths already translated into Silo's filesystem namespace (the
|
||||
// plugin has applied its own path rewrites). Files or directories.
|
||||
repeated string changed_paths = 1;
|
||||
// Opaque continuation token. The host stores it verbatim and echoes it back
|
||||
// on the next PollChanges; the host never parses it.
|
||||
string next_marker = 2;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Generate the Go code**
|
||||
|
||||
Run: `make proto`
|
||||
Expected: exits 0; creates `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go` and `scan_source_grpc.pb.go`. (If it prints `protoc is required`, install `protoc` and re-run.)
|
||||
|
||||
- [ ] **Step 3: Verify the generated symbols exist and compile**
|
||||
|
||||
Run: `go build ./... && grep -l "func NewScanSourceClient" pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go && grep -l "RegisterScanSourceServer" pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go`
|
||||
Expected: build succeeds; both `grep`s print the filename (confirming `ScanSourceClient`, `ScanSourceServer`, `NewScanSourceClient`, `RegisterScanSourceServer` were generated — `require_unimplemented_servers=false` in `buf.gen.yaml` means no forced `UnimplementedScanSourceServer` embedding).
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add proto/silo/plugin/v1/scan_source.proto pkg/pluginproto/silo/plugin/v1/scan_source.pb.go pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go
|
||||
git commit -m "feat(proto): add scan_source.v1 capability service"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Register the capability type in the allowlist
|
||||
|
||||
The manifest loader rejects capability types not in `capability.KnownTypes` (see `TestLoadRejectsUnknownCapabilityType`). A manifest declaring `scan_source.v1` cannot load until the type is registered here.
|
||||
|
||||
**Files:**
|
||||
- Modify: `pkg/pluginsdk/capability/capability.go`
|
||||
- Create or Modify: `pkg/pluginsdk/capability/capability_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Append to `pkg/pluginsdk/capability/capability_test.go` (create the file with this `package capability` header if it does not exist):
|
||||
|
||||
```go
|
||||
package capability
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestScanSourceIsKnownType(t *testing.T) {
|
||||
if ScanSource != "scan_source.v1" {
|
||||
t.Fatalf("ScanSource const = %q, want %q", ScanSource, "scan_source.v1")
|
||||
}
|
||||
found := false
|
||||
for _, k := range KnownTypes {
|
||||
if k == ScanSource {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("ScanSource (%q) missing from KnownTypes %v", ScanSource, KnownTypes)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/capability/ -run TestScanSourceIsKnownType -v`
|
||||
Expected: FAIL — `undefined: ScanSource` (the const does not exist yet).
|
||||
|
||||
- [ ] **Step 3: Add the const and allowlist entry**
|
||||
|
||||
In `pkg/pluginsdk/capability/capability.go`, add the const inside the existing `const (...)` block (after `EbookBackend`):
|
||||
|
||||
```go
|
||||
EbookBackend = "ebook_backend.v1"
|
||||
ScanSource = "scan_source.v1"
|
||||
```
|
||||
|
||||
and add it to the `KnownTypes` slice (append `ScanSource` to the existing list):
|
||||
|
||||
```go
|
||||
ScanSource,
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/capability/ -run TestScanSourceIsKnownType -v`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add pkg/pluginsdk/capability/capability.go pkg/pluginsdk/capability/capability_test.go
|
||||
git commit -m "feat(capability): register scan_source.v1 as a known capability type"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Wire the capability into the runtime
|
||||
|
||||
Add the `ScanSource` server to `CapabilityServers`, a `Client.ScanSource()` accessor, and the conditional registration in `GRPCServer` — mirroring `ScheduledTask` in `pkg/pluginsdk/runtime/runtime.go`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `pkg/pluginsdk/runtime/runtime.go`
|
||||
- Create or Modify: `pkg/pluginsdk/runtime/runtime_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Append to `pkg/pluginsdk/runtime/runtime_test.go` (create with the `package runtime` header if absent). This test constructs a real `*grpc.Server`, supplies a minimal `ScanSource` server, and asserts `GRPCServer` registers it without error:
|
||||
|
||||
```go
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type stubScanSource struct{}
|
||||
|
||||
func (stubScanSource) PollChanges(context.Context, *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
return &pluginv1.PollChangesResponse{}, nil
|
||||
}
|
||||
|
||||
func TestGRPCServerRegistersScanSource(t *testing.T) {
|
||||
p := &GRPCPlugin{Servers: CapabilityServers{
|
||||
Runtime: stubRuntime{}, // existing stub used by sibling runtime tests
|
||||
ScanSource: stubScanSource{},
|
||||
}}
|
||||
srv := grpc.NewServer()
|
||||
if err := p.GRPCServer(nil, srv); err != nil {
|
||||
t.Fatalf("GRPCServer with ScanSource = %v, want nil", err)
|
||||
}
|
||||
if _, ok := srv.GetServiceInfo()["silo.plugin.v1.ScanSource"]; !ok {
|
||||
t.Fatalf("ScanSource service not registered; got %v", srv.GetServiceInfo())
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note: `GRPCServer` requires `Servers.Runtime` to be non-nil (it errors otherwise). Reuse the existing runtime stub the sibling tests use. If no such stub exists in `runtime_test.go`, add a minimal one:
|
||||
|
||||
```go
|
||||
type stubRuntime struct{ pluginv1.RuntimeServer }
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/runtime/ -run TestGRPCServerRegistersScanSource -v`
|
||||
Expected: FAIL — `unknown field 'ScanSource' in struct literal of type CapabilityServers` (the field does not exist yet).
|
||||
|
||||
- [ ] **Step 3: Add the struct field**
|
||||
|
||||
In `pkg/pluginsdk/runtime/runtime.go`, add to the `CapabilityServers` struct (after the `ScheduledTask` field):
|
||||
|
||||
```go
|
||||
ScheduledTask pluginv1.ScheduledTaskServer
|
||||
ScanSource pluginv1.ScanSourceServer
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add the `Client` accessor**
|
||||
|
||||
After the existing `func (c *Client) ScheduledTask() ...` method, add:
|
||||
|
||||
```go
|
||||
func (c *Client) ScanSource() pluginv1.ScanSourceClient {
|
||||
return pluginv1.NewScanSourceClient(c.conn)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Add the conditional registration in `GRPCServer`**
|
||||
|
||||
In `func (p *GRPCPlugin) GRPCServer(...)`, after the `ScheduledTask` registration block, add:
|
||||
|
||||
```go
|
||||
if p.Servers.ScheduledTask != nil {
|
||||
pluginv1.RegisterScheduledTaskServer(server, p.Servers.ScheduledTask)
|
||||
}
|
||||
if p.Servers.ScanSource != nil {
|
||||
pluginv1.RegisterScanSourceServer(server, p.Servers.ScanSource)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run test to verify it passes**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/runtime/ -run TestGRPCServerRegistersScanSource -v`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add pkg/pluginsdk/runtime/runtime.go pkg/pluginsdk/runtime/runtime_test.go
|
||||
git commit -m "feat(runtime): serve and dial scan_source.v1 capability"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Prove a `scan_source.v1` manifest loads
|
||||
|
||||
Guards the end-to-end path: a plugin manifest declaring the new capability must pass the loader (it would not before Task 2's allowlist entry).
|
||||
|
||||
**Files:**
|
||||
- Modify: `pkg/pluginsdk/manifest/manifest_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
The test file is package `manifest_test` (external) and loads from a JSON literal via `manifest.Load([]byte)`. Append this test, mirroring `TestLoadAcceptsRequestRouterCapability` exactly with the type swapped to `scan_source.v1`:
|
||||
|
||||
```go
|
||||
func TestLoadAcceptsScanSourceCapability(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"plugin_id": "silo.example",
|
||||
"version": "1.0.0",
|
||||
"silo_api_version": "v1",
|
||||
"capabilities": [
|
||||
{"type": "scan_source.v1", "id": "arr", "display_name": "X", "description": "Y"}
|
||||
]
|
||||
}`)
|
||||
m, err := manifest.Load(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("Load returned unexpected error: %v", err)
|
||||
}
|
||||
if len(m.GetCapabilities()) != 1 {
|
||||
t.Fatalf("expected 1 capability, got %d", len(m.GetCapabilities()))
|
||||
}
|
||||
if got := m.GetCapabilities()[0].GetType(); got != "scan_source.v1" {
|
||||
t.Fatalf("capability type = %q, want scan_source.v1", got)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails (before this task's dependency was in place) / passes now**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/manifest/ -run TestLoadAcceptsScanSourceCapability -v`
|
||||
Expected: PASS (Task 2 already added `scan_source.v1` to `KnownTypes`). If it FAILS with an unknown-capability-type error, Task 2 was not completed correctly — fix that first.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add pkg/pluginsdk/manifest/manifest_test.go
|
||||
git commit -m "test(manifest): accept scan_source.v1 capability in manifests"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Full build + test sweep
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Build everything**
|
||||
|
||||
Run: `go build ./...`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 2: Run the full SDK test suite**
|
||||
|
||||
Run: `go test ./...`
|
||||
Expected: all packages `ok`. Confirms the additive change broke no existing capability, manifest, or runtime test.
|
||||
|
||||
- [ ] **Step 3: Vet + format**
|
||||
|
||||
Run: `go vet ./... && gofmt -l pkg proto 2>/dev/null; test -z "$(gofmt -l pkg)"`
|
||||
Expected: vet clean; `gofmt -l pkg` prints nothing.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Tag the release
|
||||
|
||||
Downstream repos (silo-server host, arr plugin) consume the SDK as a tagged module version.
|
||||
|
||||
**Files:** none (release action)
|
||||
|
||||
- [ ] **Step 1: Confirm the version to tag**
|
||||
|
||||
Check the latest tag: `git tag --list 'v*' | sort -V | tail -3`. The current consumed version is `v0.4.0`; this additive feature warrants a minor bump to **`v0.5.0`** (no breaking changes).
|
||||
|
||||
- [ ] **Step 2: Tag and push**
|
||||
|
||||
```bash
|
||||
git tag v0.5.0
|
||||
git push origin v0.5.0
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Record the contract for downstream plans**
|
||||
|
||||
Note in the host and arr-plugin plans that they must require `github.com/Silo-Server/silo-plugin-sdk v0.5.0` and use:
|
||||
- `capability.ScanSource` (`"scan_source.v1"`)
|
||||
- `runtime.CapabilityServers{ ScanSource: <impl> }` (plugin side)
|
||||
- `client.ScanSource()` → `pluginv1.ScanSourceClient.PollChanges(ctx, &pluginv1.PollChangesRequest{CapabilityId, Marker})` (host side)
|
||||
- response fields `PollChangesResponse.ChangedPaths []string`, `PollChangesResponse.NextMarker string`
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** Implements spec §5 (the `scan_source.v1` contract: `PollChanges`, opaque marker, Silo-native paths) and §11 (additive-only; registered in `KnownTypes`; no existing capability modified). Spec §12 build-order step 1 (tag the SDK) is Task 6.
|
||||
- **Out of scope here:** the host engine (spec §7), connections (§8), the Autoscan category (§9), and the arr plugin (§10) are separate per-repo plans built against the `v0.5.0` tag produced by Task 6.
|
||||
- **Marker is opaque end-to-end:** the proto carries `marker`/`next_marker` as plain strings with no server-side interpretation, satisfying the "any provider's bookmark" requirement.
|
||||
@@ -0,0 +1,265 @@
|
||||
# Sonarr/Radarr Scan-Source Plugin Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build a new installable Silo plugin that implements `scan_source.v1` for Sonarr/Radarr: when the host pulls, it polls arr `/history`, extracts imported + renamed paths, applies its own path rewrites, and returns Silo-native paths + a marker.
|
||||
|
||||
**Architecture:** A standalone Go module structured like `silo-plugin-tmdb` — a `manifest.json` declaring `scan_source.v1`, a `main.go` serving `runtime.CapabilityServers{Runtime, ScanSource}` via `runtime.Serve`, and the arr-specific logic ported from the closed silo-server PR #43 (`internal/autoscan/history.go`, `rewrite.go`, `suggest.go`).
|
||||
|
||||
**Tech Stack:** Go, `github.com/Silo-Server/silo-plugin-sdk` (`v0.5.0`, the `scan_source.v1` capability), `hashicorp/go-plugin` runtime (via the SDK), arr HTTP API (`X-Api-Key`).
|
||||
|
||||
Commands assume the **new plugin repository root** is the cwd (the `silo-plugin-autoscan-arr` checkout, a sibling of this repo like the `silo-plugin-sdk` checkout).
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **SDK `v0.5.0`.** Needs `scan_source.v1` from `silo-plugin-sdk` PR #2. Until tagged, depend on the local checkout via replace:
|
||||
```bash
|
||||
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk
|
||||
```
|
||||
This points at the `silo-plugin-sdk` checkout (a sibling of this repo). Task 8 finalizes to `v0.5.0`.
|
||||
2. **Template:** the `silo-plugin-tmdb` checkout (a sibling of this repo) — copy its `main.go` runtime scaffold, `Makefile`, and `manifest.json` shape.
|
||||
3. **Salvage source:** the arr logic is on silo-server `main` at `internal/autoscan/history.go` (imports + renames extraction, bounded window), `rewrite.go` (`applyRewrites`, `normalizeSeparators`), `suggest.go`/`suggest_deps.go` (root-folder suffix-match suggester). Port these verbatim where possible.
|
||||
4. **New repo:** create `Silo-Server/silo-plugin-autoscan-arr` (or fork pattern). The plugin is registered in the `silo-plugins` catalog in a separate follow-up (Task 9 notes it).
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `go.mod` | module `github.com/Silo-Server/silo-plugin-autoscan-arr`; SDK dep | Create |
|
||||
| `manifest.json` | declares `scan_source.v1` capability + `global_config_schema` (path rewrites only; connection arrives per request) | Create |
|
||||
| `main.go` | `runtimeServer` (GetManifest/Configure) + `scanSourceServer` (PollChanges); `runtime.Serve` | Create |
|
||||
| `internal/arr/history.go` | poll `/history`, imports + renames, bounded window | Create (port) |
|
||||
| `internal/arr/rewrite.go` | `applyRewrites`, `normalizeSeparators` | Create (port) |
|
||||
| `internal/arr/client.go` | arr HTTP client (`X-Api-Key`, response cap, timeout) | Create |
|
||||
| `internal/config/config.go` | parse Configure payload → `{rewrites}` (connection comes per request, not config) | Create |
|
||||
| `Makefile` | build (cross-platform), like tmdb | Create |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Scaffold the plugin (serves an empty `scan_source.v1`)
|
||||
|
||||
**Files:** Create `go.mod`, `manifest.json`, `main.go`, `Makefile`.
|
||||
|
||||
- [ ] **Step 1: Module + SDK dep**
|
||||
|
||||
```bash
|
||||
go mod init github.com/Silo-Server/silo-plugin-autoscan-arr
|
||||
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk # the silo-plugin-sdk checkout, a sibling of this repo
|
||||
go get github.com/Silo-Server/silo-plugin-sdk@v0.4.0 # replace points at the scan_source branch checkout
|
||||
```
|
||||
|
||||
- [ ] **Step 2: manifest.json**
|
||||
|
||||
Mirror tmdb's shape, declaring the scan source:
|
||||
|
||||
```json
|
||||
{
|
||||
"plugin_id": "silo.autoscan.arr",
|
||||
"version": "0.1.0",
|
||||
"checksum": "__CHECKSUM__",
|
||||
"silo_api_version": "v1",
|
||||
"supported_platforms": [
|
||||
{"os": "linux", "arch": "amd64"},
|
||||
{"os": "linux", "arch": "arm64"},
|
||||
{"os": "darwin", "arch": "arm64"}
|
||||
],
|
||||
"capabilities": [
|
||||
{
|
||||
"type": "scan_source.v1",
|
||||
"id": "arr",
|
||||
"display_name": "Sonarr / Radarr",
|
||||
"description": "Triggers Silo rescans from Sonarr/Radarr import and rename history."
|
||||
}
|
||||
],
|
||||
"global_config_schema": [
|
||||
{"key": "path_rewrites", "type": "json", "required": false, "display_name": "Path rewrites"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
> **Connection is NOT plugin config.** The arr base URL + API key are delivered by the host in each `PollChangesRequest.connection` (resolved from the operator's Autoscan connection — own creds or a reused Requests link). The plugin reads them from the request and stores no credentials. Only path rewrites (and other provider-specific behaviour) live in plugin config.
|
||||
|
||||
(Confirm the exact `global_config_schema` entry shape against `silo-plugin-sdk` `pkg/pluginsdk/config` — adapt field names to the SDK's config schema type.)
|
||||
|
||||
- [ ] **Step 3: main.go scaffold**
|
||||
|
||||
Copy tmdb's `runtimeServer` (GetManifest from embedded manifest; Configure stores config) and add a `scanSourceServer` returning empty for now:
|
||||
|
||||
```go
|
||||
type scanSourceServer struct{ cfg *config.Config }
|
||||
|
||||
func (s *scanSourceServer) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
return &pluginv1.PollChangesResponse{}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
manifest := mustLoadManifest()
|
||||
rt := &runtimeServer{manifest: manifest}
|
||||
runtime.Serve(runtime.ServeConfig{
|
||||
Servers: runtime.CapabilityServers{
|
||||
Runtime: rt,
|
||||
ScanSource: &scanSourceServer{cfg: rt.cfg},
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Build**
|
||||
|
||||
Run: `go build ./...`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git init && git add -A
|
||||
git commit -m "feat: scaffold autoscan-arr scan_source.v1 plugin"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Config parsing (Configure → path rewrites only)
|
||||
|
||||
**Files:** Create `internal/config/config.go`, `internal/config/config_test.go`.
|
||||
|
||||
> Connection (base_url/api_key) is NOT config — it arrives per poll in `PollChangesRequest.connection` (Task 5). Plugin config carries only path rewrites.
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
|
||||
```go
|
||||
func TestParseConfig(t *testing.T) {
|
||||
cfg, err := Parse(map[string]string{
|
||||
"path_rewrites": `[{"from":"/mnt/arr/tv","to":"/mnt/media/tv"}]`,
|
||||
})
|
||||
if err != nil { t.Fatalf("Parse: %v", err) }
|
||||
if len(cfg.Rewrites) != 1 || cfg.Rewrites[0].From != "/mnt/arr/tv" { t.Fatalf("bad rewrites: %+v", cfg.Rewrites) }
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → fail.** `go test ./internal/config/` → undefined `Parse`.
|
||||
|
||||
- [ ] **Step 3: Implement** `Parse(map[string]string) (*Config, error)` with `Config{Rewrites []Rewrite}` where `Rewrite{From, To string}`; unmarshal `path_rewrites` JSON. Wire `runtimeServer.Configure` to store the parsed config so `scanSourceServer` reads the rewrites.
|
||||
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
|
||||
- [ ] **Step 5: Commit** `feat(config): parse arr connection and path rewrites`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: arr history client (imports + renames, bounded window)
|
||||
|
||||
**Files:** Create `internal/arr/client.go`, `internal/arr/history.go`, `internal/arr/history_test.go`.
|
||||
|
||||
Port from silo-server `main:internal/autoscan/history.go` (the `ChangedPaths` logic: `downloadFolderImported.importedPath`; `episodeFileRenamed`/`movieFileRenamed` → `path` + `sourcePath`; deletes ignored) and the client (`X-Api-Key`, 1 MiB cap, timeout). The marker is an RFC3339 timestamp; empty marker ⇒ "now".
|
||||
|
||||
- [ ] **Step 1: Failing test** — copy the salvaged `history_test.go` (`TestArrHistoryChangedPaths`: imports + both rename paths returned, `grabbed`/`episodeFileDeleted` ignored), renamed to this package, asserting the returned `(paths, nextMarker)`.
|
||||
|
||||
- [ ] **Step 2: Run → fail.**
|
||||
|
||||
- [ ] **Step 3: Implement** `ChangedPaths(ctx, baseURL, apiKey, since time.Time) (paths []string, newest time.Time, err error)` — port `history.go`, additionally returning the newest history timestamp as the next marker. Apply the bounded window (24h max-lookback floor + overlap buffer) here.
|
||||
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
|
||||
- [ ] **Step 5: Commit** `feat(arr): poll history for imports and renames`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Path rewrites (→ Silo-native paths)
|
||||
|
||||
**Files:** Create `internal/arr/rewrite.go`, `internal/arr/rewrite_test.go`.
|
||||
|
||||
Port `applyRewrites` (boundary-safe prefix: `path==trimmed || HasPrefix(path, trimmed+"/")`) and `normalizeSeparators` from silo-server `main:internal/autoscan/rewrite.go`.
|
||||
|
||||
- [ ] **Step 1: Failing test** — copy salvaged `rewrite_test.go` cases.
|
||||
- [ ] **Step 2: Run → fail.**
|
||||
- [ ] **Step 3: Implement** the port.
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
- [ ] **Step 5: Commit** `feat(arr): boundary-safe path rewrites`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Wire `PollChanges` end-to-end
|
||||
|
||||
**Files:** Modify `main.go` (`scanSourceServer.PollChanges`), add `main_test.go`.
|
||||
|
||||
- [ ] **Step 1: Failing test** — a `scanSourceServer` (rewrites configured) called with a `PollChangesRequest` whose `Connection` points at a stub arr server (httptest); assert `PollChanges` with empty marker returns rewritten Silo-native paths and a non-empty `next_marker`, and that the request actually hit the stub at `connection.base_url` with `X-Api-Key: connection.api_key`. A second call with the returned marker queries arr with the right `since`. Also assert a request with a nil/empty `Connection` returns an error (no credentials to poll with).
|
||||
|
||||
- [ ] **Step 2: Run → fail.**
|
||||
|
||||
- [ ] **Step 3: Implement** — read the connection from the request, not from config:
|
||||
|
||||
```go
|
||||
func (s *scanSourceServer) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
conn := req.GetConnection()
|
||||
if conn == nil || conn.GetBaseUrl() == "" {
|
||||
return nil, fmt.Errorf("scan_source: no connection supplied")
|
||||
}
|
||||
since := time.Time{} // empty marker => now (history client floors to now)
|
||||
if m := req.GetMarker(); m != "" {
|
||||
if t, err := time.Parse(time.RFC3339, m); err == nil { since = t }
|
||||
}
|
||||
raw, newest, err := arr.ChangedPaths(ctx, conn.GetBaseUrl(), conn.GetApiKey(), since)
|
||||
if err != nil { return nil, err }
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, p := range raw {
|
||||
out = append(out, arr.ApplyRewrites(arr.NormalizeSeparators(p), s.cfg.Rewrites))
|
||||
}
|
||||
return &pluginv1.PollChangesResponse{ChangedPaths: out, NextMarker: newest.UTC().Format(time.RFC3339)}, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
|
||||
- [ ] **Step 5: Commit** `feat: PollChanges returns Silo-native paths from arr history`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Rewrite suggester ("Sync from arr") — root folders
|
||||
|
||||
**Files:** Create `internal/arr/suggest.go`, `suggest_test.go`.
|
||||
|
||||
Port `suggestRewrites` (suffix-match arr root folders → Silo folders) from `main:internal/autoscan/suggest.go`. **Open question (spec §14 risk #2):** how the operator triggers "Sync" and how the plugin reaches Silo's media-folder list.
|
||||
|
||||
- [ ] **Step 1:** Decide the delivery mechanism — recommended: the plugin reads Silo's folder list via the host library-listing service (`pluginhost` library lister, reachable through the SDK `runtime.Host()` client). Confirm that path exists in `v0.5.0`; if not, scope "Sync" to a follow-up and ship Tasks 1–5 (manual rewrites only) first.
|
||||
- [ ] **Step 2:** Port the suffix-match suggester + its tests (unique/ambiguous/covered/no-op/normalization cases from `suggest_test.go`).
|
||||
- [ ] **Step 3:** Expose it — either via an `http_routes.v1` capability the admin UI calls, or as part of the plugin config flow. Document the chosen mechanism.
|
||||
- [ ] **Step 4:** Tests pass; commit `feat(arr): suffix-match rewrite suggester`.
|
||||
|
||||
> If §14 risk #2 is unresolved at execution time, **skip this task**, ship manual rewrites (Tasks 1–5), and file the suggester as a follow-up. The plugin is fully functional without it.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Cross-repo smoke test against the host
|
||||
|
||||
**Files:** none (verification).
|
||||
|
||||
- [ ] **Step 1:** Build the plugin binary; install it into a dev silo-server (with the Part-1 backend) via the plugin installer.
|
||||
- [ ] **Step 2:** Configure a source pointing at a real/stub arr; trigger an autoscan poll; confirm the host enqueues a scan for an imported path. (This exercises the full SDK contract end-to-end.)
|
||||
- [ ] **Step 3:** Note results; no commit unless fixes are needed.
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Finalize SDK dependency
|
||||
|
||||
- [ ] After `silo-plugin-sdk v0.5.0` is tagged: `go mod edit -dropreplace ...; go get .../silo-plugin-sdk@v0.5.0; go mod tidy; go build ./...`.
|
||||
- [ ] Commit `build: depend on silo-plugin-sdk v0.5.0`.
|
||||
- [ ] Do not publish/tag the plugin with a `replace` directive in `go.mod`.
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Catalog registration (follow-up, separate repo)
|
||||
|
||||
- [ ] Add `silo.autoscan.arr` to the `silo-plugins` catalog manifest so it is installable. This is a separate PR in the `silo-plugins` repo (not this module) — note it; out of scope for this plan's repo.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** §10 (arr plugin: history imports + renames, bounded window, rewrites, Silo-native paths) Tasks 3–5; §5 contract (`PollChanges`, opaque marker = RFC3339 timestamp, first-run "now") Task 5; "Sync from arr" §10/§14-risk-2 Task 6 (with an explicit skip path).
|
||||
- **Salvage:** history/rewrite/suggest ported from the closed PR (Tasks 3,4,6) — exactly the files the host backend plan deletes (Part-1 Task 8).
|
||||
- **Risk:** §14 risk #1 (one plugin installation per arr server vs. many) — this plugin holds one connection per installation/config; multiple arr servers ⇒ multiple installs unless the runtime supports multiple capability instances. §14 risk #2 (Sync UI) gated in Task 6 with a clean skip.
|
||||
- **Type consistency:** `Config{Rewrites}` (no creds — connection arrives in `PollChangesRequest.connection`), `Rewrite{From,To}`, `arr.ChangedPaths(ctx, baseURL, apiKey, since) ([]string, time.Time, error)`, `arr.ApplyRewrites`/`NormalizeSeparators`, `scanSourceServer.PollChanges` consistent across Tasks 2–6.
|
||||
@@ -0,0 +1,209 @@
|
||||
# Autoscan Admin UI Implementation Plan (Part 2 of 2)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Give Autoscan its own admin category (page + sidebar nav), separate from Requests: manage Connections (reuse a Requests arr server or enter own), Sources (per scan-source plugin: enable, interval, connection), and global settings — backed by the Part-1 v2 API.
|
||||
|
||||
**Architecture:** Extract the Autoscan tab out of `AdminRequests.tsx` into a standalone `AdminAutoscan.tsx` page; register a route and a sidebar item; rebuild `useAutoscan.ts` hooks + `types.ts` against the v2 endpoints (`/admin/autoscan/{settings,connections,sources,trigger,status}`).
|
||||
|
||||
**Tech Stack:** React + TypeScript, React Query (`@tanstack/react-query`), the project's `ui/` component library (`Tabs`, `Table`, `Card`, etc.), `react-router` routes in `App.tsx`, 2-space indent / double quotes / 100-col (`web/.prettierrc`).
|
||||
|
||||
Commands assume the silo-server repository root is the cwd. **Depends on Part 1 (backend)** being implemented (the v2 endpoints must exist).
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Part-1 backend merged/available** — the v2 endpoints and response shapes (connections with `request_integration_id`, sources with `marker`/`last_error`, settings with `default_poll_interval_seconds`).
|
||||
2. **Reference the existing autoscan tab** at `web/src/pages/AdminRequests.tsx` (the `autoscan` `TabsContent`, the `AutoscanTab`/`AutoscanSourceEditor` components) and hooks at `web/src/hooks/queries/useAutoscan.ts` — these are the starting material; they are reshaped, not discarded.
|
||||
3. **Run before MR:** `cd web && pnpm run lint && pnpm run format:check`. Note the 20 pre-existing failing frontend tests are unrelated (project memory); do not be alarmed if they appear — only newly-introduced failures matter.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `web/src/api/types.ts` | `AutoscanConnection`, `AutoscanSource`, `AutoscanSettings` (v2 shapes) | Modify |
|
||||
| `web/src/hooks/queries/useAutoscan.ts` | hooks for settings/connections/sources/trigger/status | Rewrite |
|
||||
| `web/src/hooks/queries/keys.ts` | query keys for connections/sources | Modify |
|
||||
| `web/src/pages/AdminAutoscan.tsx` | the new standalone Autoscan page (Connections + Sources + Settings tabs) | Create |
|
||||
| `web/src/pages/admin/autoscan/ConnectionsPanel.tsx` | list/add/edit/delete connections (own or Requests-linked) | Create |
|
||||
| `web/src/pages/admin/autoscan/SourcesPanel.tsx` | per-source enable, interval, connection binding, status | Create |
|
||||
| `web/src/pages/AdminRequests.tsx` | remove the Autoscan tab | Modify |
|
||||
| `web/src/App.tsx` | add `<Route path="autoscan" element={<AdminAutoscan />} />` | Modify |
|
||||
| `web/src/components/AdminSidebar.tsx` | add the Autoscan nav item under "Content" | Modify |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: v2 types + hooks
|
||||
|
||||
**Files:** Modify `web/src/api/types.ts`, `web/src/hooks/queries/keys.ts`; rewrite `web/src/hooks/queries/useAutoscan.ts`.
|
||||
|
||||
- [ ] **Step 1: Types**
|
||||
|
||||
Add to `web/src/api/types.ts`:
|
||||
|
||||
```ts
|
||||
export interface AutoscanSettings {
|
||||
enabled: boolean;
|
||||
default_poll_interval_seconds: number;
|
||||
debounce_seconds: number;
|
||||
}
|
||||
|
||||
export interface AutoscanConnection {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
// present when the connection has its own credentials:
|
||||
base_url?: string;
|
||||
// present when linked to a Requests integration (live reuse):
|
||||
request_integration_id?: string | null;
|
||||
// NOTE: api_key_ref / resolved keys are never sent by the backend.
|
||||
}
|
||||
|
||||
export interface AutoscanSource {
|
||||
id: string;
|
||||
installation_id: number;
|
||||
capability_id: string;
|
||||
connection_id: string;
|
||||
enabled: boolean;
|
||||
poll_interval_seconds: number | null;
|
||||
last_run_at: string | null;
|
||||
last_error: string | null;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Hooks**
|
||||
|
||||
Rewrite `useAutoscan.ts` with: `useAutoscanSettings`/`useUpdateAutoscanSettings`, `useAutoscanConnections`/`useCreateAutoscanConnection`/`useUpdateAutoscanConnection`/`useDeleteAutoscanConnection`, `useAutoscanSources`/`useUpdateAutoscanSource`, `useAutoscanStatus`, `useTriggerAutoscan`. Follow the existing hook style (the current `useAutoscan.ts` is the template for query/mutation + invalidation). Add query keys in `keys.ts`.
|
||||
|
||||
- [ ] **Step 3: Typecheck**
|
||||
|
||||
Run: `cd web && pnpm exec tsc --noEmit`
|
||||
Expected: no type errors in the autoscan files.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add web/src/api/types.ts web/src/hooks/queries/useAutoscan.ts web/src/hooks/queries/keys.ts
|
||||
git commit -m "feat(web): autoscan v2 types and query hooks"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Connections panel
|
||||
|
||||
**Files:** Create `web/src/pages/admin/autoscan/ConnectionsPanel.tsx`.
|
||||
|
||||
- [ ] **Step 1: Build the panel**
|
||||
|
||||
A `Card` + `Table` listing connections (name, kind, and a "Reused from Requests" / "Own" badge derived from `request_integration_id`). An "Add connection" dialog offers two modes:
|
||||
- **Reuse from Requests** — a select populated from the existing request-integrations hook (filter to arr kinds), storing `request_integration_id`.
|
||||
- **Enter own** — name + URL + API key fields, posting own credentials.
|
||||
|
||||
Edit + delete actions per row. Never render any key material (the backend doesn't send it).
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck**
|
||||
|
||||
Run: `cd web && pnpm run lint && pnpm exec tsc --noEmit`
|
||||
Expected: clean.
|
||||
|
||||
- [ ] **Step 3: Commit** `feat(web): autoscan connections panel (reuse or own)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Sources panel
|
||||
|
||||
**Files:** Create `web/src/pages/admin/autoscan/SourcesPanel.tsx`.
|
||||
|
||||
- [ ] **Step 1: Build the panel**
|
||||
|
||||
A `Table` of sources, one row per installed `scan_source` capability instance: plugin/capability label, a connection `<select>` (bound to `connection_id`), an enable toggle, a poll-interval input (blank = settings default), and a status column (`last_run_at`, `last_error`). Reuse the spinner/"Syncing…" affordances from the old `AutoscanSourceEditor` where relevant. Wire to `useAutoscanSources`/`useUpdateAutoscanSource`.
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck** — clean.
|
||||
- [ ] **Step 3: Commit** `feat(web): autoscan sources panel`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: The Autoscan page + global settings + manual trigger
|
||||
|
||||
**Files:** Create `web/src/pages/AdminAutoscan.tsx`.
|
||||
|
||||
- [ ] **Step 1: Compose the page**
|
||||
|
||||
A page with a `Tabs` (or sections): **Sources**, **Connections**, **Settings**. Settings = global enable toggle, default poll interval, debounce. A "Run now" button calling `useTriggerAutoscan` (202 → toast "Autoscan triggered"). Mirror the header/layout of `AdminRequests.tsx` for visual consistency.
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck** — clean.
|
||||
- [ ] **Step 3: Commit** `feat(web): standalone Autoscan admin page`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Route + sidebar nav
|
||||
|
||||
**Files:** Modify `web/src/App.tsx`, `web/src/components/AdminSidebar.tsx`.
|
||||
|
||||
- [ ] **Step 1: Route**
|
||||
|
||||
In `web/src/App.tsx`, next to `<Route path="requests" element={<AdminRequests />} />` (l.374), add:
|
||||
|
||||
```tsx
|
||||
<Route path="autoscan" element={<AdminAutoscan />} />
|
||||
```
|
||||
|
||||
with the matching `import AdminAutoscan from "@/pages/AdminAutoscan";`.
|
||||
|
||||
- [ ] **Step 2: Sidebar item**
|
||||
|
||||
In `web/src/components/AdminSidebar.tsx`, under the **Content** group (after the Requests item, ~l.112), add:
|
||||
|
||||
```tsx
|
||||
{
|
||||
label: "Autoscan",
|
||||
icon: <RefreshCw className="h-[18px] w-[18px]" />,
|
||||
href: "/admin/autoscan",
|
||||
},
|
||||
```
|
||||
|
||||
(Import an appropriate icon from the icon set already used in the file.)
|
||||
|
||||
- [ ] **Step 3: Verify navigation**
|
||||
|
||||
Run: `cd web && pnpm run build`
|
||||
Expected: build succeeds. Manually confirm `/admin/autoscan` renders and the sidebar item highlights.
|
||||
|
||||
- [ ] **Step 4: Commit** `feat(web): route and sidebar nav for Autoscan category`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Remove the Autoscan tab from Requests
|
||||
|
||||
**Files:** Modify `web/src/pages/AdminRequests.tsx`.
|
||||
|
||||
- [ ] **Step 1: Excise the tab**
|
||||
|
||||
Remove the `<TabsTrigger value="autoscan">` (l.146) and its `<TabsContent value="autoscan">`, the `AutoscanTab`/`AutoscanSourceEditor` definitions (now superseded by the new panels), and `autoscan` from `ADMIN_REQUEST_TABS`. Remove now-unused imports.
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck/build**
|
||||
|
||||
Run: `cd web && pnpm run lint && pnpm run format:check && pnpm run build`
|
||||
Expected: clean; build succeeds; no dangling references to removed components.
|
||||
|
||||
- [ ] **Step 3: Commit** `refactor(web): move Autoscan out of Requests into its own category`.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Final sweep
|
||||
|
||||
- [ ] **Step 1:** `cd web && pnpm run lint && pnpm run format:check && pnpm run build` — all clean.
|
||||
- [ ] **Step 2:** `cd web && pnpm test` — confirm no **newly** failing tests (the 20 pre-existing failures, per project memory, are unrelated; compare against baseline).
|
||||
- [ ] **Step 3:** Manual pass: add a connection (both modes), bind a source, toggle enable, "Run now", confirm status updates.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** §9 (Autoscan as its own category, decoupled from Requests; Connections reuse-or-own; per-source enable/interval/status) — Tasks 1–6. The reuse-from-Requests UI (Task 2) surfaces the soft link from spec §8.
|
||||
- **Salvage:** the old `AutoscanTab`/`AutoscanSourceEditor` and `useAutoscan.ts` are reshaped (Tasks 1–3), then the Requests tab is removed (Task 6).
|
||||
- **Out of scope:** plugin-side config UI (the path-rewrite table lives on the plugin's own settings screen per spec §10 — that is the arr-plugin plan / spec §14 risk #2, not this page).
|
||||
- **Type consistency:** `AutoscanConnection{request_integration_id?}`, `AutoscanSource{connection_id,poll_interval_seconds,last_error}`, `AutoscanSettings{default_poll_interval_seconds}` used consistently across hooks (Task 1) and panels (Tasks 2–4).
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,174 @@
|
||||
# Audiobooks Stacked PR Split Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace the oversized audiobook PR with a stacked series of smaller, reviewable PRs.
|
||||
|
||||
**Architecture:** Preserve the existing `feat/audiobooks` work as the source branch, then create branch cut points that expose one logical layer at a time. Each PR targets the previous branch so reviewers see only the incremental diff.
|
||||
|
||||
**Tech Stack:** Git, GitHub CLI, Go, pnpm/Vite, PostgreSQL migrations.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Close The Oversized PR
|
||||
|
||||
**Files:**
|
||||
- Modify: GitHub PR #17 only.
|
||||
|
||||
- [ ] **Step 1: Post a replacement note**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr comment 17 --repo Silo-Server/silo-server --body 'Closing this oversized PR in favor of a stacked series of smaller PRs. The existing branch is preserved as the source-of-truth while the stack is rebuilt into logical review units.'
|
||||
```
|
||||
|
||||
Expected: GitHub prints the created comment URL.
|
||||
|
||||
- [ ] **Step 2: Close PR #17 without deleting the branch**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr close 17 --repo Silo-Server/silo-server
|
||||
```
|
||||
|
||||
Expected: PR #17 state becomes `CLOSED`; branch `feat/audiobooks` remains available.
|
||||
|
||||
### Task 2: Create Stacked Branches
|
||||
|
||||
**Files:**
|
||||
- Modify: remote Git branches only.
|
||||
|
||||
- [ ] **Step 1: Create branch 1, foundation**
|
||||
|
||||
Branch name: `stack/audiobooks-foundation`
|
||||
|
||||
Scope:
|
||||
- audiobook feature flag/settings
|
||||
- audiobook and podcast schema foundations
|
||||
- scanner support
|
||||
- audiobook media item write path
|
||||
- no native UI
|
||||
- no ABS listener
|
||||
|
||||
- [ ] **Step 2: Create branch 2, native UI MVP**
|
||||
|
||||
Branch name: `stack/audiobooks-native-ui`
|
||||
|
||||
Base: `stack/audiobooks-foundation`
|
||||
|
||||
Scope:
|
||||
- `/api/v1/audiobooks`
|
||||
- audiobook detail/progress endpoints
|
||||
- audiobook frontend route/sidebar/player MVP
|
||||
|
||||
- [ ] **Step 3: Create branch 3, ABS core**
|
||||
|
||||
Branch name: `stack/audiobooks-abs-core`
|
||||
|
||||
Base: `stack/audiobooks-native-ui`
|
||||
|
||||
Scope:
|
||||
- ABS auth/login/refresh/logout
|
||||
- ABS listener and feature-flag listener gating
|
||||
- ABS library/items/play/progress
|
||||
- ABS access control and security hardening
|
||||
|
||||
- [ ] **Step 4: Create branch 4, ABS collections**
|
||||
|
||||
Branch name: `stack/audiobooks-abs-collections`
|
||||
|
||||
Base: `stack/audiobooks-abs-core`
|
||||
|
||||
Scope:
|
||||
- ABS bookmarks
|
||||
- ABS collections
|
||||
- ABS playlists
|
||||
- ABS smart collections
|
||||
- unified collection migration
|
||||
|
||||
- [ ] **Step 5: Create branch 5, extras and polish**
|
||||
|
||||
Branch name: `stack/audiobooks-extras`
|
||||
|
||||
Base: `stack/audiobooks-abs-collections`
|
||||
|
||||
Scope:
|
||||
- podcasts/RSS
|
||||
- stats
|
||||
- author/series extras
|
||||
- catalog audiobook filters/typeahead
|
||||
- performance and enrichment cleanup
|
||||
|
||||
### Task 3: Open The Stacked PRs
|
||||
|
||||
**Files:**
|
||||
- Modify: GitHub PRs only.
|
||||
|
||||
- [ ] **Step 1: Open PR 1**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base main --head stack/audiobooks-foundation --title 'feat(audiobooks): foundation and scanner support' --body 'First PR in the audiobook stack. Adds the schema/settings/scanner foundation without native UI or ABS compatibility.'
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Open PR 2**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-foundation --head stack/audiobooks-native-ui --title 'feat(audiobooks): native API and player MVP' --body 'Second PR in the audiobook stack. Adds native Silo audiobook endpoints and the MVP web player UI.'
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Open PR 3**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-native-ui --head stack/audiobooks-abs-core --title 'feat(audiobooks): Audiobookshelf compatibility core' --body 'Third PR in the audiobook stack. Adds ABS auth, listener, library/items/play/progress, and security hardening.'
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Open PR 4**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-abs-core --head stack/audiobooks-abs-collections --title 'feat(audiobooks): ABS collections, playlists, and smart collections' --body 'Fourth PR in the audiobook stack. Adds ABS bookmarks, collections, playlists, smart collections, and unified collection storage.'
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Open PR 5**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-abs-collections --head stack/audiobooks-extras --title 'feat(audiobooks): podcasts, catalog filters, and enrichment polish' --body 'Final PR in the audiobook stack. Adds podcast/RSS extras, audiobook catalog filters/typeahead, and scan/enrichment performance polish.'
|
||||
```
|
||||
|
||||
### Task 4: Verify And Report
|
||||
|
||||
**Files:**
|
||||
- Modify: none.
|
||||
|
||||
- [ ] **Step 1: Verify PR states**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr list --repo Silo-Server/silo-server --state open --search 'audiobooks in:title' --json number,title,headRefName,baseRefName,url
|
||||
```
|
||||
|
||||
Expected: five open stacked PRs with the base/head chain shown above.
|
||||
|
||||
- [ ] **Step 2: Run branch verification where feasible**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
cd web && pnpm run build
|
||||
```
|
||||
|
||||
Expected: both commands pass on the final stack tip.
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# Resilient Library Deletion Design
|
||||
|
||||
## Goal
|
||||
|
||||
Make `delete_library` survive concurrent database writers and large libraries. Today it deletes an entire library inside a single multi-minute transaction; on a large library this deadlocks against ordinary background writes and rolls back, making zero progress. Rewrite the deletion to run in small, autocommitted batches with per-batch deadlock retry, so it completes reliably, holds only short locks, and resumes cleanly if interrupted.
|
||||
|
||||
## Background
|
||||
|
||||
A `delete_library` admin job repeatedly failed with:
|
||||
|
||||
```
|
||||
deleting orphaned items: ERROR: deadlock detected (SQLSTATE 40P01)
|
||||
```
|
||||
|
||||
Diagnosis on a library of ~249K items / ~339K media files:
|
||||
|
||||
- The whole deletion runs in one transaction in `FolderRepository.DeleteWithStats` (`internal/catalog/folder_repo.go`): collect orphan IDs, drop the folder (CASCADE), then `DELETE FROM media_items WHERE content_id = ANY($1)`.
|
||||
- `media_items` cascades to 15 child tables, so a bulk delete holds a very large lock set for minutes.
|
||||
- The Postgres deadlock log shows the cycle: the delete cascade into `media_item_provider_ids` versus a concurrent `INSERT INTO media_item_provider_ids` from a routine background task. Each ~7-minute attempt was guaranteed to overlap an ordinary write and lose.
|
||||
- Because it is one transaction, every failure rolls back completely: no progress, not resumable.
|
||||
|
||||
The fix targets the deletion mechanism only. Coordinating or pausing background writers is explicitly out of scope (see Out of Scope) — a batched, retrying delete is robust against *any* concurrent writer, not just today's tasks.
|
||||
|
||||
## Approach
|
||||
|
||||
Replace the single transaction with a sequence of phases. Each phase deletes in bounded batches, and each batch is one autocommit statement wrapped in a deadlock-retry helper. Short lock windows make deadlocks rare; when one occurs, only that small batch retries.
|
||||
|
||||
Scope of the rewrite:
|
||||
|
||||
- Rewrite `FolderRepository.DeleteWithStats` in `internal/catalog/folder_repo.go`.
|
||||
- Add an unexported `retryOnDeadlock` helper in the `catalog` package.
|
||||
- `FolderRepository.Delete` (thin wrapper), the `DeleteFolderStats` struct, and the `delete_library` executor in `internal/adminjob/library_delete.go` keep their current signatures and behavior. S3 image-cleanup queueing and generated-home-section cleanup are untouched.
|
||||
|
||||
## Deletion phases
|
||||
|
||||
`DeleteWithStats(ctx, id, progress)` performs, in order:
|
||||
|
||||
### Phase 0 — preflight (reads only)
|
||||
|
||||
- Load the folder name; return `ErrFolderNotFound` if the row is gone.
|
||||
- `COUNT` media files (`WHERE media_folder_id = $1`), item links (`WHERE media_folder_id = $1`), and orphans, to populate the result payload and the progress denominator.
|
||||
|
||||
### Phase 1 — orphaned `media_items` (batched loop)
|
||||
|
||||
Repeat until no rows are returned:
|
||||
|
||||
```sql
|
||||
SELECT mil.content_id
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.media_folder_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries other
|
||||
WHERE other.content_id = mil.content_id
|
||||
AND other.media_folder_id <> $1
|
||||
)
|
||||
LIMIT $2 -- orphanDeleteBatch
|
||||
```
|
||||
|
||||
For each returned batch: collect S3 image dirs for those IDs (existing `collectImageDirs`, accumulated into a deduped set), then
|
||||
|
||||
```sql
|
||||
DELETE FROM media_items WHERE content_id = ANY($1)
|
||||
```
|
||||
|
||||
The FK cascade removes each item's `media_item_libraries` (including this folder's), `media_item_provider_ids`, episodes, seasons, embeddings, and the rest of the 15 child tables. Deleting orphan items first also clears their junction rows, so the next iteration's detection query naturally advances. Orphan detection must run before Phase 3 touches shared junctions, so the "only in this folder" test stays valid.
|
||||
|
||||
### Phase 2 — folder `media_files` (batched loop)
|
||||
|
||||
`media_files` is folder-tied (`media_folder_id` FK, `ON DELETE CASCADE`), not item-tied, so it is deleted by folder. Repeat until a batch deletes fewer than the batch size:
|
||||
|
||||
```sql
|
||||
DELETE FROM media_files
|
||||
WHERE id IN (
|
||||
SELECT id FROM media_files WHERE media_folder_id = $1 LIMIT $2 -- folderChildDeleteBatch
|
||||
)
|
||||
```
|
||||
|
||||
### Phase 3 — remaining shared-item junctions (batched loop)
|
||||
|
||||
After Phase 1, the only `media_item_libraries` rows left for this folder belong to items that also live in other folders. Delete them in batches (dropping the membership only, never the shared item):
|
||||
|
||||
```sql
|
||||
DELETE FROM media_item_libraries
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM media_item_libraries WHERE media_folder_id = $1 LIMIT $2 -- folderChildDeleteBatch
|
||||
)
|
||||
```
|
||||
|
||||
### Phase 4 — folder row
|
||||
|
||||
```sql
|
||||
DELETE FROM media_folders WHERE id = $1
|
||||
```
|
||||
|
||||
Now lightweight. Tolerates 0 rows affected (a resumed re-run may already have removed it) — it does not re-raise `ErrFolderNotFound` here. The folder row is deleted **last**, so a partial deletion leaves the library present-but-disabled and a re-run continues.
|
||||
|
||||
## Deadlock-retry helper
|
||||
|
||||
New unexported helper in the `catalog` package:
|
||||
|
||||
```go
|
||||
func retryOnDeadlock(ctx context.Context, op func() error) error
|
||||
```
|
||||
|
||||
- Retries when the error is a `*pgconn.PgError` with code `40P01` (deadlock) or `40001` (serialization failure).
|
||||
- Up to 5 attempts, exponential backoff starting at 50ms, doubling each retry.
|
||||
- Honors `ctx` cancellation between attempts (returns `ctx.Err()`).
|
||||
- Any non-retryable error returns immediately; the final retryable error is returned after the cap.
|
||||
|
||||
Every batch statement runs through `retryOnDeadlock`. It has a single caller today, so it stays package-local; it can be promoted to a shared DB utility if a second caller appears.
|
||||
|
||||
## Batch sizes
|
||||
|
||||
Named constants in `internal/catalog/folder_repo.go`, tunable:
|
||||
|
||||
- `orphanDeleteBatch = 1000` — small, because each `media_items` row cascades across 15 tables.
|
||||
- `folderChildDeleteBatch = 5000` — for the lighter folder-scoped `media_files` and junction deletes.
|
||||
|
||||
## Transaction model, stats, progress
|
||||
|
||||
- No surrounding transaction. Each batch is a single `pool.Exec` (its own implicit transaction), mirroring the existing batched-cleanup pattern in `internal/opslog/cleanup.go`. This is the core change that eliminates the multi-minute lock window.
|
||||
- Stats: `MediaFiles` and `MediaItemLinks` come from the Phase 0 counts; `OrphanedItems` accumulates across Phase 1 batches; `OrphanedImageDirs` is the accumulated deduped dir set. Image-dir volume for the affected workloads is modest; the deduped set is the bounded-memory point of note.
|
||||
- Progress: the callback reports orphan items processed against the Phase 0 orphan total, plus phase-transition messages. The admin-job runner's existing 10s heartbeat keeps the job alive across the drain.
|
||||
|
||||
## Error handling & resumability
|
||||
|
||||
- Because batches autocommit, a failure or cancellation persists partial progress. There is no rollback of hundreds of thousands of rows and no long lock hold.
|
||||
- The job is resumable by re-running it: the folder still exists (deleted last), and Phase 1 re-detects whatever orphans remain.
|
||||
- New items inserted concurrently during deletion are picked up by the loop's re-detection.
|
||||
|
||||
## Testing
|
||||
|
||||
The `catalog` package uses pure-Go unit tests; there is no Postgres integration harness, so tests avoid a live database.
|
||||
|
||||
- **`retryOnDeadlock` unit test:** a fake `op` returns a `*pgconn.PgError{Code: "40P01"}` N times then `nil` — assert it retries, respects the attempt cap, backs off, and returns immediately on a non-retryable error; a separate case asserts `ctx` cancellation stops retries.
|
||||
- **Batch-loop unit test:** extract the "loop until a batch is under the batch size" iteration behind a small injected exec function so the looping/chunking logic is testable without Postgres.
|
||||
- **Integration verification on dev:** deploy and re-trigger the deletion against the real stuck library; confirm it drains to completion via application logs and database row counts. This is the integration test, given the absence of a catalog DB harness. (Commands assume the repository root is the cwd and follow the deployment-debugging runbook.)
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Pausing or coordinating background writers (e.g., having metadata/scan tasks skip disabled or pending-deletion libraries). The batched, retrying delete is robust against any concurrent writer; writer coordination is a separate potential improvement.
|
||||
- Changing the admin-job runner, the executor's image-cleanup queueing, or home-section cleanup.
|
||||
- Schema or FK changes.
|
||||
@@ -0,0 +1,141 @@
|
||||
# Calendar Presets & Personalized Default — Design
|
||||
|
||||
- **Date:** 2026-05-29
|
||||
- **Status:** Approved (design); ready for implementation planning
|
||||
- **Scope:** `silo-server` (Go backend + web admin UI). Client follow-ups in `silo-android` / `silo-apple` are additive and optional.
|
||||
- **Note:** All paths are repository-relative. Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
The calendar works but is not practical. It defaults to *every* upcoming airing in the catalog, so it is noisy and undifferentiated — the user has to mentally filter out shows they don't care about. We want a Simkl-style preset switcher with a personalized default, powered by the server's real watch data rather than a manually curated list.
|
||||
|
||||
## Goals
|
||||
|
||||
- Default the calendar to a personalized **Following** view (shows the profile actually engages with).
|
||||
- Offer a small set of presets, including a **server-wide** popularity view and an **external trending** view.
|
||||
- Keep it cheap under load (hundreds of concurrent users): presets must **filter, not compute**.
|
||||
- Make "what do I still need to watch this week" visible via a watched overlay.
|
||||
- Preserve backward compatibility for existing clients.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No cross-user caching of the windowed airing query yet (phased — add only if metrics show strain).
|
||||
- No pulling in external content that isn't in the library (e.g. a followed show whose next season hasn't been added). The calendar stays catalog-scoped.
|
||||
- No change to the week-view layout, timezone handling, or event ordering (recently tuned; left intact).
|
||||
- No region/country or "For Kids" presets (considered and cut).
|
||||
|
||||
## Presets
|
||||
|
||||
The selector replaces today's `All / Favorites / Watchlist` toggles.
|
||||
|
||||
| Preset | Shows | Source |
|
||||
|---|---|---|
|
||||
| **Following** *(default)* | Upcoming airings for series the profile has watched, favorited, or watchlisted | per-profile: `user_favorites` ∪ `user_watchlist` ∪ watched-series rollup |
|
||||
| **Popular here** | Airings for what is most-watched server-wide | cached `recommendation_cache` (`RecTypePopular`, global key) |
|
||||
| **Trending** | Airings for externally-trending shows that exist in the library | `trending_discover_snapshots` |
|
||||
| **Everything** | All catalog airings (today's behavior) | no restriction |
|
||||
|
||||
Per-profile vs server-wide is split cleanly across two presets: **Following** is personal (this profile), **Popular here** is the server-wide aggregate.
|
||||
|
||||
## Core principle: filter, don't compute
|
||||
|
||||
Every preset resolves to **the windowed airing candidate list ∩ a small id-set**. No preset triggers a fresh aggregation at request time.
|
||||
|
||||
1. **Step 1 — windowed candidate fetch** (unchanged from today): the 3-branch UNION in `internal/catalog/calendar_repo.go` over movies/episodes/season-premieres, scoped only by library access + content rating, across the ±2-day-padded window. Its result depends only on the access signature, not on viewer identity, so it remains cacheable across users later.
|
||||
2. **Step 2 — intersect with one id-set**, chosen by preset:
|
||||
- **Following** → resolve the profile's followed id-set once per request, then filter `itemIDExpr = ANY($set)`.
|
||||
- **Popular / Trending** → read the already-cached global id-set and intersect.
|
||||
- **Everything** → no restriction.
|
||||
3. **Step 3 — group by day, sort by viewer-local wall-clock time, respond** (unchanged).
|
||||
|
||||
The only heavy SQL is Step 1, which already runs today for every calendar load. Presets add a bounded id-set intersection, never a new aggregation.
|
||||
|
||||
### Unify the filter mechanism
|
||||
|
||||
All personal/preset filters resolve to "an id-set, then `itemIDExpr = ANY($set)`". This replaces the special-cased favorites/watchlist `EXISTS` subqueries in `appendPersonalFilterClause` with a single code path (favorites and watchlist become id-sets resolved from their tables — they are subsets of Following). One mechanism is easier to reason about and removes duplicated clause-building.
|
||||
|
||||
`itemIDExpr` continues to be `mi.content_id` for movies and `e.series_id` / `s.series_id` for episodes / season premieres, so a watched/favorited **movie** matches by its own content_id while a followed **series** matches by series id.
|
||||
|
||||
### Following id-set
|
||||
|
||||
Resolved per request as the DISTINCT union of:
|
||||
|
||||
- `SELECT media_item_id FROM user_favorites WHERE user_id = $u AND profile_id = $p`
|
||||
- `SELECT media_item_id FROM user_watchlist WHERE user_id = $u AND profile_id = $p`
|
||||
- `SELECT DISTINCT COALESCE(e.series_id, wp.media_item_id) FROM user_watch_progress wp LEFT JOIN episodes e ON e.content_id = wp.media_item_id WHERE wp.user_id = $u AND wp.profile_id = $p`
|
||||
|
||||
"Engaged" = the presence of a `user_watch_progress` row (you've started it); no completion threshold required for Following. The rollup mirrors the established pattern in `GetPopularItems` (`internal/recommendations/repo.go`) — reuse that rollup expression rather than duplicate it.
|
||||
|
||||
The candidate list (Step 1) is small (one week of airings), so intersecting against a bounded array is cheap. Caveat: a pathologically large followed-set (a profile that has watched thousands of distinct series) makes the array large; if that ever shows up as hot, switch the Following preset to correlated `EXISTS` against the user's tables. Not building that now.
|
||||
|
||||
### Popular / Trending id-sets
|
||||
|
||||
- **Popular**: `GetRecommendationCache(ctx, GlobalCacheUserID, GlobalCacheProfileID, RecTypePopular, "")` → `[]ScoredItem`; use `MediaItemID` values (already series-level via the `GetPopularItems` rollup). Refreshed by the recommendations worker on a TTL; readers never aggregate live.
|
||||
- **Trending**: `TrendingSnapshotRepository.Get(ctx, source, window)` → `TrendingSnapshot.ContentIDs` (pre-resolved to catalog content_ids, viewer-agnostic). The calendar reads the canonical default snapshot (`tmdb` / `week`). Unioning multiple sources/windows is possible but out of scope.
|
||||
|
||||
**Access is preserved automatically:** Popular/Trending id-sets are viewer-agnostic and may include items a given profile cannot access. Because we intersect them with the access-scoped Step-1 candidates (library scoping + `MaxContentRating` already applied there), inaccessible items drop out naturally. No extra access check needed on the id-sets.
|
||||
|
||||
**Cold/empty cache:** if the Popular cache or Trending snapshot is empty (fresh server, worker hasn't run, provider unconfigured), the id-set is empty and the preset returns no rows. This must render the empty state, not an error.
|
||||
|
||||
## Watched overlay
|
||||
|
||||
After Step 1, decorate the windowed events with per-profile watched status:
|
||||
|
||||
- One bounded lookup: `SELECT media_item_id FROM user_watch_progress WHERE user_id = $u AND profile_id = $p AND completed = true AND media_item_id = ANY($episodeContentIds)` over the window's episode/movie content_ids.
|
||||
- Watched items render dimmed + ✓. This is a per-profile overlay applied after Step 1 (like the Following filter), so it does not affect the shareability of the Step-1 candidate set.
|
||||
- Premiere / finale / new-season badges extend the existing `buildBadges` logic in `internal/api/handlers/calendar.go`.
|
||||
|
||||
## UI / UX
|
||||
|
||||
- **Selector** (`web/src/pages/Calendar.tsx`): segmented pills on desktop, collapsing to a dropdown on narrow viewports — the same responsive swap the week navigator already uses. The library picker stays beside it. Both presentations bind to one selected-preset state.
|
||||
- **Empty state**: when **Following** has no upcoming airings (new or sparse profile), show a nudge with quick switches to Popular / Trending / Everything instead of a blank week.
|
||||
- **Persistence**: remember the profile's last-chosen preset in `localStorage` (keyed by profile id) so the calendar reopens where the user left it. Server-side profile preference is a possible future enhancement.
|
||||
- **Card cues** (`web/src/components/calendar/CalendarEventCard.tsx`, `web/src/lib/upcomingEventPresentation.ts`): badges + watched overlay as above.
|
||||
|
||||
## API contract
|
||||
|
||||
- Extend `/calendar`'s `filter` parameter to accept `following | popular | trending | everything`.
|
||||
- **Keep `all | favorites | watchlist` valid** so existing `silo-android` / `silo-apple` clients keep working unchanged. `all` and `everything` are equivalent. This is purely additive.
|
||||
- Add an additive `watched: bool` field (and any new badge string) to each event item. Older clients ignore unknown fields.
|
||||
- Default when `filter` is omitted: today it is `all`. The **web client** will request `following` explicitly; the server default stays `all` to avoid changing behavior for existing API consumers. (Default-view personalization is a client choice, not a server-contract change.)
|
||||
- **Profile requirement:** the `/calendar` route already runs behind `RequireProfile`, so a profile is always present. `following` (and legacy `favorites` / `watchlist`) plus the watched overlay are per-profile and resolve against it; `popular` / `trending` / `everything` / `all` are profile-agnostic and ignore it.
|
||||
|
||||
### Multi-repo coordination (per CLAUDE.md)
|
||||
|
||||
- `silo-android` / `silo-apple`: may adopt the new presets and the watched cue as independent follow-ups. Nothing here forces a lockstep change because old filter values and response shape remain valid.
|
||||
|
||||
## Implementation surface
|
||||
|
||||
Backend:
|
||||
- `internal/catalog/calendar_repo.go` — add an id-set restriction to `CalendarFilter`; generalize `appendPersonalFilterClause` into a single `ANY($set)` path; keep library/rating clauses intact.
|
||||
- A followed-id-set resolver, `ListFollowedSeriesIDs(ctx, userID, profileID)`, placed in the package that owns the favorites/watchlist/episode join; reuse the `GetPopularItems` rollup expression.
|
||||
- `internal/api/handlers/calendar.go` — validate the expanded `filter` set; orchestrate preset → id-set resolution (Following resolver, Popular cache read, Trending snapshot read); apply the watched overlay; extend `buildBadges`.
|
||||
|
||||
Frontend:
|
||||
- `web/src/pages/Calendar.tsx` — responsive selector + preset persistence + empty state.
|
||||
- `web/src/hooks/queries/calendar.ts` — new `filter` values; `watched` field on the event type.
|
||||
- `web/src/components/calendar/CalendarEventCard.tsx` — watched overlay + badge rendering.
|
||||
- `web/src/lib/upcomingEventPresentation.ts` — badge/treatment helpers.
|
||||
|
||||
## Edge cases
|
||||
|
||||
- Cold/empty Popular cache or Trending snapshot → empty preset → empty state, never an error.
|
||||
- Profile with no follows → Following empty → empty state with preset nudges.
|
||||
- Large followed-set → bounded array today; correlated-`EXISTS` fallback noted for later.
|
||||
- Movie vs episode/season id matching (`content_id` vs `series_id`) handled by the existing `itemIDExpr` selection per branch.
|
||||
- Specials (`season_number = 0`) remain excluded.
|
||||
- Access control (`MaxContentRating`, library scoping, disabled libraries) still enforced for every preset via Step 1; viewer-agnostic id-sets cannot leak inaccessible items because of the intersection.
|
||||
- Backward-compat filter values (`all | favorites | watchlist`) still resolve through the unified id-set path with identical results.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
- **Repo** (`internal/catalog/calendar_repo_test.go`): each preset's id-set filter (following / popular / trending / everything), empty id-set → zero rows, access + rating enforcement preserved across presets, movie-by-content_id vs series-by-series_id matching, legacy favorites/watchlist parity.
|
||||
- **Handler** (`internal/api/handlers/calendar_test.go`): filter validation accepts new + legacy values; Popular/Trending cold cache degrades to empty gracefully; `watched` field populated correctly; watched lookup bounded to the window.
|
||||
- **Frontend**: selector responsive swap (pills ↔ dropdown), preset persistence across reloads, watched overlay + badge rendering, Following empty state.
|
||||
|
||||
## Future work
|
||||
|
||||
- Cross-user cache of the Step-1 windowed candidate set keyed by (window, access-signature), if load metrics justify it.
|
||||
- External enrichment of followed shows whose next season is not yet in the library.
|
||||
- Server-side persisted preset preference per profile.
|
||||
- Optional union of multiple Trending sources/windows.
|
||||
@@ -0,0 +1,208 @@
|
||||
# Trending Discover — Persistent Snapshot Design
|
||||
|
||||
Status: Approved (design)
|
||||
Date: 2026-05-29
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
The `trending_discover` home section pulls global trending (TMDB or Trakt),
|
||||
matches it to titles in the viewer's libraries, and renders the result. Today
|
||||
the external fetch + external-ID resolution are memoised in an **in-process
|
||||
map** (`editorialCandidateCache`) keyed by `source|window|fetchLimit` with a
|
||||
1-hour TTL and a `singleflight` group to collapse concurrent misses
|
||||
(`internal/sections/fetcher.go`, `loadTrendingDiscoverContentIDs`).
|
||||
|
||||
That cache has three weaknesses, two of which we explicitly want to fix:
|
||||
|
||||
1. **No resilience to upstream failure.** At TTL expiry a request still blocks
|
||||
on the TMDB/Trakt call, and a slow or down provider degrades the home page.
|
||||
There is no concept of "serve the last good list."
|
||||
2. **No observability.** Nothing records when trending last refreshed, whether
|
||||
it succeeded, or how many trending titles matched the catalog.
|
||||
3. (Secondary) It is per-process and volatile — lost on restart, and each
|
||||
server instance fetches independently.
|
||||
|
||||
## Goals
|
||||
|
||||
- Reads of the trending section **never call the upstream provider** and never
|
||||
block on it. They serve a persisted, last-good list.
|
||||
- A persisted, inspectable record with refresh history (the same "synced
|
||||
record" shape used by library collections).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Reusing the library-collection machinery. `library_collections.library_id` is
|
||||
`NOT NULL` (collections are library-scoped); trending is deliberately
|
||||
library-agnostic and appears once. Making collections library-agnostic is a
|
||||
large, separate change and is out of scope.
|
||||
- An admin UI for the snapshot. The persisted columns make this possible later,
|
||||
but it is not part of this work.
|
||||
- A shared/distributed cache (Redis). The DB is the single source of truth.
|
||||
|
||||
## Approach
|
||||
|
||||
Replace the lazy in-process cache with a **background-refreshed persistent
|
||||
snapshot**. A scheduled task refreshes the trending list on an interval; the
|
||||
read path only ever reads the persisted snapshot. This mirrors the
|
||||
collection-sync *pattern* (`CollectionSyncScheduler` + `SyncCollectionsTask`)
|
||||
without reusing its library-scoped storage.
|
||||
|
||||
The change cleanly separates **read** from **refresh**:
|
||||
|
||||
- The refresh path owns the upstream call and external-ID resolution.
|
||||
- The read path owns only a primary-key snapshot lookup plus the existing
|
||||
per-viewer access filtering.
|
||||
|
||||
## Data model
|
||||
|
||||
New migration `167_trending_discover_snapshots`. (Originally numbered 166, but
|
||||
the shared dev DB had already recorded version 166 from another branch's
|
||||
`166_trending_blend_collection_type`; the migration runner dedupes by integer
|
||||
version, so 166 was silently skipped. Renumbered to 167, the next free version.)
|
||||
|
||||
```
|
||||
trending_discover_snapshots
|
||||
source text NOT NULL -- 'tmdb' | 'trakt'
|
||||
window text NOT NULL -- 'day' | 'week' (trakt canonicalized to 'week')
|
||||
content_ids text[] NOT NULL DEFAULT '{}' -- ordered, resolved to library catalog, capped at 200
|
||||
entry_count int NOT NULL DEFAULT 0 -- raw provider entries fetched (matched-vs-trending observability)
|
||||
refreshed_at timestamptz -- last successful refresh
|
||||
last_attempt_at timestamptz
|
||||
last_status text -- 'ok' | 'empty' | 'error'
|
||||
last_error text
|
||||
PRIMARY KEY (source, window)
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- One row per canonical `(source, window)` — at most a handful of rows.
|
||||
- `content_ids` is catalog-matched but **viewer-agnostic**, exactly like the
|
||||
value the current cache holds. Per-viewer access filtering still happens at
|
||||
read time in `fetchItemsByContentIDs`, so a single row serves all viewers.
|
||||
- The list is stored at the over-fetch cap (200). The read path truncates to the
|
||||
section's `ItemLimit`, decoupling the snapshot from per-section limits (the old
|
||||
`fetchLimit` cache-key dimension goes away).
|
||||
- **Reliability invariant:** a failed refresh updates `last_attempt_at`,
|
||||
`last_status`, and `last_error` but **never clears `content_ids`**. The last
|
||||
good list keeps serving through an upstream outage.
|
||||
|
||||
The paired `.down.sql` drops the table.
|
||||
|
||||
## Components
|
||||
|
||||
All new server code lives in `internal/`.
|
||||
|
||||
### `TrendingSnapshotRepository` (new, `internal/sections/`)
|
||||
|
||||
Thin repository over the new table:
|
||||
|
||||
- `Get(ctx, source, window) (TrendingSnapshot, error)` — PK lookup.
|
||||
- `Upsert(ctx, snapshot) error` — insert/update on `(source, window)`.
|
||||
- `ListAll(ctx) ([]TrendingSnapshot, error)` — for future admin/inspection use
|
||||
and tests.
|
||||
|
||||
`Upsert` distinguishes a successful refresh (writes `content_ids`,
|
||||
`refreshed_at`, `entry_count`, `last_status='ok'|'empty'`) from a failed one
|
||||
(touches only `last_attempt_at`/`last_status='error'`/`last_error`, leaving
|
||||
`content_ids` intact).
|
||||
|
||||
### `TrendingRefresher` (new, `internal/sections/trending_refresher.go`)
|
||||
|
||||
Owns the upstream call. Holds `TMDBTrending`, `TraktTrending`, `ItemRepo`, the
|
||||
section repository (to enumerate used combos), and the snapshot repository. The
|
||||
existing `fetchTrendingDiscoverEntries` and `resolveTrendingDiscoverIDs` logic
|
||||
**moves here** from `fetcher.go`.
|
||||
|
||||
`RunOnce(ctx) (json.RawMessage, error)`:
|
||||
|
||||
1. Enumerate the distinct canonical `(source, window)` pairs from **enabled**
|
||||
`trending_discover` sections across all scopes (parse each section's
|
||||
`TrendingDiscoverParams`, apply the same source/window normalization the
|
||||
section uses, collapse Trakt to `week`).
|
||||
2. For each pair: fetch the cap-200 list from the provider, resolve external IDs
|
||||
to library content IDs, and `Upsert`.
|
||||
3. Return a JSON summary (e.g. `{combos, refreshed, empty, failed}`), mirroring
|
||||
`CollectionSyncResult`.
|
||||
|
||||
No used combos → no upstream calls (a dormant feature costs nothing). A
|
||||
configured-but-empty provider yields `last_status='empty'` (no error noise). A
|
||||
per-pair failure is recorded and does not abort the other pairs.
|
||||
|
||||
### `RefreshTrendingDiscoverTask` (new, `internal/taskmanager/tasks/`)
|
||||
|
||||
Mirrors `SyncCollectionsTask`: delegates to `TrendingRefresher.RunOnce` and
|
||||
reports progress/result data. Default trigger: hourly interval (matches the
|
||||
recipe's `DefaultCacheTTL` of 1h), plus a one-shot run at startup so the first
|
||||
snapshot lands quickly rather than after a full interval.
|
||||
|
||||
### `Fetcher` (modified, `internal/sections/fetcher.go`)
|
||||
|
||||
- `loadTrendingDiscoverContentIDs` now reads the snapshot row via the snapshot
|
||||
repository and returns its `content_ids`. It drops the upstream fetch, the
|
||||
`singleflight` group, and the `editorialCandidateCache` usage **for trending**
|
||||
(the editorial-candidate cache remains for the editorial sections that still
|
||||
use it).
|
||||
- `fetchTrendingDiscover` keeps its read-time responsibilities: access
|
||||
filtering, re-ordering to trending rank, truncation to `ItemLimit`. It no
|
||||
longer computes a `fetchLimit` for the read path.
|
||||
- The `TMDBTrending`, `TraktTrending`, and trending-only use of `ItemRepo`
|
||||
migrate out of the read path into the refresher. The fetcher gains the
|
||||
snapshot repository dependency.
|
||||
|
||||
### Wiring (`cmd/silo/main.go`, `internal/api/router.go`)
|
||||
|
||||
- Construct `TrendingSnapshotRepository` and `TrendingRefresher` (wired with the
|
||||
TMDB/Trakt fetchers, `ItemRepo`, section repo, snapshot repo).
|
||||
- Register `RefreshTrendingDiscoverTask` alongside the other task registrations.
|
||||
- Hand the snapshot repository to the section `Fetcher`; move the upstream
|
||||
fetchers from the fetcher wiring to the refresher wiring.
|
||||
|
||||
## Data flow
|
||||
|
||||
**Refresh (background, hourly + startup):**
|
||||
task → `Refresher.RunOnce` → enumerate distinct canonical `(source, window)`
|
||||
from enabled `trending_discover` sections → per pair: fetch cap-200 → resolve
|
||||
external IDs → `Upsert` snapshot.
|
||||
|
||||
**Read (per home render):**
|
||||
`fetchTrendingDiscover` → `loadTrendingDiscoverContentIDs` reads the snapshot
|
||||
row → `fetchItemsByContentIDs` applies the viewer's access filter → re-order to
|
||||
trending rank → truncate to `ItemLimit`. No upstream call. A single PK lookup on
|
||||
a ≤6-row table — negligible next to the queries `FetchAll` already runs, so no
|
||||
in-process read cache is added (it would re-muddy the single-source-of-truth and
|
||||
is premature).
|
||||
|
||||
## Edge behavior
|
||||
|
||||
- **Before first sync:** snapshot absent → section renders empty (no blocking,
|
||||
no upstream call). The startup task run closes this gap to roughly one fetch
|
||||
duration.
|
||||
- **Provider unconfigured but a section exists:** `last_status='empty'`, no
|
||||
error.
|
||||
- **Upstream failure on a pair that already has a snapshot:** prior
|
||||
`content_ids` keep serving; only the attempt/status/error columns update.
|
||||
- **Trakt `day` vs `week`:** collapse to a single `week` row (Trakt ignores the
|
||||
window).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Repository round-trip:** upsert then get; `content_ids` ordering preserved;
|
||||
success vs failure upsert paths.
|
||||
- **Refresher `RunOnce`:** fake `TMDBTrending`/`TraktTrending`, fake section
|
||||
enumerator, in-memory `ItemRepo`. Assert ordered IDs upserted; `ok`/`empty`/
|
||||
`error` transitions; and the reliability invariant — a failed refresh
|
||||
preserves prior `content_ids`.
|
||||
- **Read path:** inject nil upstream fetchers into the fetcher and assert the
|
||||
trending read still works from the snapshot (proves reads never call upstream).
|
||||
- **Canonicalization:** Trakt `day` and `week` resolve to one row.
|
||||
|
||||
## Risks / follow-ups
|
||||
|
||||
- The data captured (`refreshed_at`, `last_status`, `last_error`, `entry_count`)
|
||||
enables a future admin/inspection surface; not built here.
|
||||
- If multiple sections request very different `ItemLimit`s, all share the cap-200
|
||||
list and truncate — intended, and strictly more data than the old per-limit
|
||||
cache held.
|
||||
```
|
||||
@@ -0,0 +1,353 @@
|
||||
# Multi-Instance Sonarr/Radarr Routing for Requests (Seerr-style)
|
||||
|
||||
**Status:** Design approved, ready for implementation plan
|
||||
**Date:** 2026-06-01
|
||||
**Area:** `internal/requests`, `migrations/`, `web/src` (admin settings + request queue)
|
||||
|
||||
## Summary
|
||||
|
||||
Silo's request system today fulfills every approved request through a single
|
||||
Radarr (movies) or single Sonarr (series) instance, at whatever quality that
|
||||
one instance's profile is configured for. There is no 4K concept, no anime
|
||||
handling, and exactly one instance per kind (`request_integrations.kind` is the
|
||||
primary key).
|
||||
|
||||
This work replicates Seerr's Sonarr/Radarr management model *inside* Silo's own
|
||||
request system (no external Seerr dependency):
|
||||
|
||||
- **Many instances per kind** — N Radarr and N Sonarr servers, each with its own
|
||||
connection, root folder, quality profile, and tags.
|
||||
- **Default HD + Default 4K routing** — one HD-default and one 4K-default
|
||||
instance per kind; requests route to the matching default(s).
|
||||
- **Per-server anime overrides** (Overseerr-style) — each instance carries an
|
||||
optional anime profile / root folder / tags applied when a title is detected
|
||||
as anime.
|
||||
- **Entitlement-driven dual-quality fan-out** — a single user request can be
|
||||
fulfilled in **both** 1080p and 2160p when the requester's `MaxPlaybackQuality`
|
||||
allows 4K, or when an admin global toggle forces it for everyone.
|
||||
|
||||
The end-user request action is unchanged: quality is derived from the user's
|
||||
existing entitlement, not a per-request choice.
|
||||
|
||||
## Goals
|
||||
|
||||
- Support multiple Radarr/Sonarr instances per kind.
|
||||
- Route by quality (HD vs 4K) to per-kind default instances.
|
||||
- Fan a single request out to both 1080p and 2160p targets based on the
|
||||
requester's `MaxPlaybackQuality` entitlement, with an admin override that
|
||||
forces dual-quality regardless of role.
|
||||
- Apply per-instance anime overrides using Seerr-exact anime detection.
|
||||
- Keep the request lifecycle (quotas, approval, reconcile) coherent under
|
||||
one-to-many fulfillment.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No integration *with* a Seerr instance (an earlier idea, dropped). Silo
|
||||
remains the system of record.
|
||||
- No per-request manual server/profile picker (Overseerr's advanced UI). Routing
|
||||
is automatic from defaults + entitlement.
|
||||
- No separate 1080p/4K *availability* tracking. The pre-request presence check
|
||||
stays binary ("is this title in the Silo library at all").
|
||||
- No admin-configurable anime keyword list (the heuristic is isolated so one can
|
||||
be added later).
|
||||
- No dedicated-anime-instance flag — anime is handled by per-instance overrides
|
||||
(decided during design).
|
||||
|
||||
## Background: current flow
|
||||
|
||||
When a user requests a movie today (`internal/requests/service.go`):
|
||||
|
||||
1. **Browse/discover** via TMDB; each result carries a `RequestState`.
|
||||
2. `CreateRequest` validates: requests enabled, not already available
|
||||
(presence), not already actively requested (dedup by `tmdb_id`), computes the
|
||||
requester's `EffectivePolicy` (quota, blocked, auto-approve), enforces quota.
|
||||
3. Auto-approve users with a configured integration → row created `approved`,
|
||||
else `pending` for the admin queue.
|
||||
4. `submitApprovedRequest` → `integrationForMediaType(mediaType)` finds the
|
||||
single instance by kind → `radarr.Client.SubmitMovie` POSTs to Radarr using
|
||||
that instance's root folder / quality profile / tags → request goes `queued`.
|
||||
5. `ReconcileRequests` polls the instance: `queued → downloading → completed`
|
||||
(or `failed`).
|
||||
|
||||
The single insertion point that changes is step 4's instance selection.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Data model
|
||||
|
||||
**`request_integrations` — reworked from one-row-per-kind to many instances:**
|
||||
|
||||
| column | purpose |
|
||||
|---|---|
|
||||
| `id` (text PK, via `idgen`) | replaces `kind` as PK |
|
||||
| `kind` (text, `radarr`\|`sonarr`) | now a plain column, keeps its CHECK |
|
||||
| `name` (text) | admin-facing label |
|
||||
| `enabled` (bool) | |
|
||||
| `base_url`, `api_key_ref` | connection; API key stays a Fernet secret ref, never plaintext |
|
||||
| `root_folder`, `quality_profile_id`, `tags[]` | standard (non-anime) defaults |
|
||||
| `is_4k` (bool) | marks this as a 4K server |
|
||||
| `is_default` (bool) | the HD default for its kind |
|
||||
| `is_default_4k` (bool) | the 4K default for its kind |
|
||||
| `anime_enabled` (bool) | apply anime overrides on this instance |
|
||||
| `anime_quality_profile_id`, `anime_root_folder`, `anime_tags[]` | anime overrides |
|
||||
| `options` (jsonb) | kind-specific extras (Radarr min-availability, Sonarr season-folders/series-type) |
|
||||
| `last_check_*`, `created_at`, `updated_at` | unchanged |
|
||||
|
||||
**Invariants** (partial unique indexes + service validation):
|
||||
|
||||
- At most one `is_default` per kind; at most one `is_default_4k` per kind.
|
||||
- `is_default ⇒ NOT is_4k` and `is_default_4k ⇒ is_4k`. The HD default is an HD
|
||||
server; the 4K default is a 4K server. HD and 4K defaults are therefore always
|
||||
distinct instances.
|
||||
|
||||
**New `media_request_targets`:**
|
||||
|
||||
| column | purpose |
|
||||
|---|---|
|
||||
| `id` (bigint identity PK) | |
|
||||
| `request_id` (text → `media_requests` ON DELETE CASCADE) | |
|
||||
| `integration_id` (text → `request_integrations` ON DELETE SET NULL) | |
|
||||
| `integration_kind` (text snapshot) | survives instance deletion for history |
|
||||
| `quality` (text, `1080p`\|`2160p`) | reuses `internal/access` presets |
|
||||
| `is_anime` (bool) | which profile set was applied |
|
||||
| `external_id`, `external_status`, `status`, `last_error` | per-target Radarr/Sonarr lifecycle |
|
||||
| `created_at`, `updated_at` | |
|
||||
| UNIQUE `(request_id, quality)` | one target per quality per request |
|
||||
|
||||
**`media_requests` changes:** the per-fulfillment columns
|
||||
(`integration_kind`, `external_id`, `external_status`) move out to
|
||||
`media_request_targets`. Add `is_anime` (bool). The request's `status` becomes an
|
||||
**aggregate** over its targets (see §3). `idx_media_requests_active_tmdb` stays —
|
||||
dedup is still one active request per title regardless of quality, and a
|
||||
dual-quality request counts as **one** row against the user's quota.
|
||||
|
||||
### 2. Routing engine
|
||||
|
||||
A pure function `routeTargets(req, entitlement, settings, instances) -> []plannedTarget`
|
||||
replaces `integrationForMediaType` as the selection point in
|
||||
`submitApprovedRequest`.
|
||||
|
||||
**Inputs:** the request (incl. detected `is_anime`), the requester's
|
||||
`MaxPlaybackQuality`, the global `force_dual_quality` setting, the configured
|
||||
instances for the kind.
|
||||
|
||||
**Algorithm:**
|
||||
|
||||
1. **Kind** = `movie → radarr`, `series → sonarr`.
|
||||
2. **Desired qualities:**
|
||||
- `1080p` is always desired (baseline for everyone).
|
||||
- Add `2160p` if `access.QualityAllowed("2160p", user.MaxPlaybackQuality)`
|
||||
**OR** `force_dual_quality` is on (the toggle overrides role).
|
||||
3. **Pick the default instance per desired quality:** `1080p → is_default`,
|
||||
`2160p → is_default_4k` (for that kind).
|
||||
4. **Profile selection per chosen instance:** if `req.is_anime` **and** the
|
||||
instance has `anime_enabled` → use its anime profile / root folder / tags
|
||||
(and, for Sonarr, `seriesType: anime`); otherwise the standard set.
|
||||
5. Emit one `plannedTarget{instance, quality, isAnime, profile, rootFolder, tags}`
|
||||
per resolved quality.
|
||||
|
||||
**Edge cases:**
|
||||
|
||||
- A desired quality with **no default instance assigned** is silently skipped
|
||||
(cannot route 4K without a 4K default — matches "when both are assigned").
|
||||
- **Zero** resolved targets (e.g. no HD default configured): the request stays
|
||||
`approved` but unfulfilled, with `last_error` = "no Radarr/Sonarr instance
|
||||
configured for this quality" so it surfaces in the admin queue.
|
||||
- No dedup needed: the `is_default`/`is_default_4k` invariants guarantee HD and
|
||||
4K defaults are distinct instances.
|
||||
|
||||
### 3. Fulfillment & reconcile lifecycle
|
||||
|
||||
The adapter interfaces (`SubmitMovie`/`SubmitSeries`,
|
||||
`CheckMovieStatus`/`CheckSeriesStatus`) are **unchanged** — they already take
|
||||
`(req, integration)`. We call them **once per target**, passing a *resolved*
|
||||
`Integration` whose `RootFolder`/`QualityProfileID`/`Tags` are filled from the
|
||||
standard or anime block. Radarr/Sonarr's notion of "4K" is just that instance's
|
||||
quality profile + root folder, so adapters stay quality-agnostic. (The Sonarr
|
||||
adapter additionally sets `seriesType: anime` for anime targets.)
|
||||
|
||||
**On approval (`submitApprovedRequest`):**
|
||||
|
||||
1. `routeTargets(...)` → planned targets.
|
||||
2. For each: insert a `media_request_targets` row, then call the matching
|
||||
adapter with the resolved instance + decrypted key.
|
||||
3. On success → target `queued` with its `external_id`/`external_status`; on
|
||||
error → target `failed` + `last_error`.
|
||||
4. **Recompute the request aggregate** in the same transaction as the target
|
||||
writes.
|
||||
|
||||
**Reconcile loop (`ReconcileRequests`):** for each candidate request, iterate its
|
||||
**non-terminal targets**, load each target's instance by `integration_id`, call
|
||||
the status adapter, update the target, then recompute the aggregate. Targets
|
||||
reconcile independently.
|
||||
|
||||
**Aggregate recompute (single source of truth):**
|
||||
|
||||
- all targets `completed` → request `completed` (+ `completed_at`)
|
||||
- any target `downloading` → `downloading`; else any `queued` → `queued`
|
||||
- all targets `failed` → `outcome = failed`; **some** failed while others active
|
||||
→ request stays active, `last_error` set from the failed target(s)
|
||||
|
||||
**Retry** is target-aware: an admin retry re-submits only the `failed` targets
|
||||
(re-routing if instance config changed), leaving healthy targets alone.
|
||||
|
||||
### 4. Anime detection (Seerr-exact)
|
||||
|
||||
Detection matches upstream Seerr exactly — a single TMDB keyword id, no genre or
|
||||
language fallback:
|
||||
|
||||
- `server/api/themoviedb/constants.ts`: `ANIME_KEYWORD_ID = 210024`
|
||||
- `server/entity/MediaRequest.ts`: anime ⇔ `keywords.results` contains that id.
|
||||
|
||||
Implementation:
|
||||
|
||||
- A named constant `animeKeywordID = 210024`.
|
||||
- `detectAnime(detail) bool` → true iff the TMDB keyword **ids** contain
|
||||
`210024`. Matching by id (not the name string) avoids false positives from
|
||||
neighboring keywords like "based on anime".
|
||||
- Detected once at `CreateRequest` time and stored on `media_requests.is_anime`,
|
||||
so routing, reconcile, and retries stay consistent.
|
||||
- Applies to both movies and series; ignored if no instance for that kind has
|
||||
`anime_enabled`.
|
||||
|
||||
**Supporting TMDB-client change:** extend `tmdb.MediaDetail` to carry keyword
|
||||
**ids** (add `append_to_response=keywords` to the detail fetch). `GetMediaDetail`
|
||||
is already called in the request path, so no extra round-trip beyond the append.
|
||||
(`OriginalLanguage` is not needed under the parity-only decision but may be
|
||||
mapped opportunistically since it is already in the response.)
|
||||
|
||||
The heuristic is isolated in one function so an admin-configurable keyword list
|
||||
can be added later without touching routing.
|
||||
|
||||
### 5. Admin UI
|
||||
|
||||
`web/src/pages/admin-settings/IntegrationsSettings.tsx` and the setup wizard
|
||||
`web/src/pages/setup-wizard/steps/IntegrationsStep.tsx` move from a single
|
||||
Radarr/Sonarr form to an **instance-list manager** per kind:
|
||||
|
||||
**Per kind — list of instance cards** (add / edit / delete):
|
||||
|
||||
- Connection: name, base URL, API key (write-only; shows "configured" once set,
|
||||
never echoes the secret), **Test connection** → `LoadIntegrationOptions`
|
||||
populates root-folder and quality-profile **dropdowns**.
|
||||
- Standard block: root folder, quality profile, tags.
|
||||
- Quality role: `is_4k` switch, **Default (HD)** toggle, **Default 4K** toggle.
|
||||
The form enforces invariants client-side (no Default-HD on a 4K server;
|
||||
selecting a new default clears the prior one visually); server is source of
|
||||
truth.
|
||||
- Anime block (collapsible, `anime_enabled`): anime quality profile, anime root
|
||||
folder, anime tags — dropdowns from the same test-connection options.
|
||||
|
||||
**Request settings panel** gains the **"Always fulfill in both 1080p and 4K"**
|
||||
toggle (`force_dual_quality`) with helper text: applies to all requests when both
|
||||
a Default HD and Default 4K instance exist, regardless of user role.
|
||||
|
||||
**`web/src/pages/AdminRequests.tsx`** (queue): each request row expands to show
|
||||
its **targets** — quality badge (1080p/2160p), instance name, per-target status,
|
||||
and per-target **retry** on failed ones. The aggregate status stays the headline.
|
||||
|
||||
**Setup wizard** stays minimal: add one Radarr + one Sonarr, auto-marked Default
|
||||
HD. Advanced multi-instance/4K/anime config lives in the full settings page.
|
||||
|
||||
### 6. API contract & multi-repo coordination
|
||||
|
||||
**End-user request flow is unchanged** — quality is derived from
|
||||
`MaxPlaybackQuality`, so the request action and payload are identical (no 4K
|
||||
toggle, no new permission).
|
||||
|
||||
**`Request` response model** gains `is_anime` and a `targets[]` array
|
||||
(`quality`, `instance_name`, `status`, `external_status`, `last_error`); the
|
||||
top-level `integration_kind`/`external_id`/`external_status` are removed in favor
|
||||
of the aggregate `status` + `targets`.
|
||||
|
||||
- Per `CLAUDE.md`'s multi-repo rule: end-user `silo-android` / `silo-apple`
|
||||
clients show a single aggregate status, so they likely need **no change** —
|
||||
but this is flagged as explicit Apple/Android follow-up to verify the request
|
||||
model tolerates the removed fields and ignores/parses the new `targets`.
|
||||
|
||||
**Admin endpoints** change from "upsert by kind" to instance CRUD:
|
||||
|
||||
- `GET/POST/PUT/DELETE` request integrations **by `id`**; `LoadIntegrationOptions`
|
||||
(test-connection) stays, keyed by instance.
|
||||
- Default-toggle handled server-side transactionally (a new HD/4K default clears
|
||||
the prior one for that kind).
|
||||
- `request_settings` gains `force_dual_quality`.
|
||||
- Admin-web-only; no mobile client surface.
|
||||
|
||||
### 7. Migration & backfill
|
||||
|
||||
Next migration number: **169** (`169_request_multi_instance.{up,down}.sql`).
|
||||
|
||||
**Up:**
|
||||
|
||||
1. `ALTER request_integrations`: add `id`, `name`, `is_4k`, `is_default`,
|
||||
`is_default_4k`, `anime_enabled`, `anime_quality_profile_id`,
|
||||
`anime_root_folder`, `anime_tags[]`. Backfill existing rows: `id = gen`,
|
||||
`name = initcap(kind)`, `is_default = enabled` (the lone existing instance
|
||||
becomes that kind's HD default), everything else default/false. Swap the PK
|
||||
from `kind` to `id`; keep `kind` as a plain column with its CHECK.
|
||||
2. Partial unique indexes: one `is_default` per kind, one `is_default_4k` per
|
||||
kind.
|
||||
3. `CREATE media_request_targets` (per §1).
|
||||
4. Backfill targets: for any `media_requests` row with an `external_id`, insert a
|
||||
target `(quality='1080p', is_anime=false, integration_id = that kind's
|
||||
instance, integration_kind, external_id, external_status, status = request's
|
||||
current status)`. Rows never submitted get no target yet.
|
||||
5. `ALTER media_requests`: add `is_anime bool DEFAULT false`; drop
|
||||
`integration_kind`, `external_id`, `external_status`.
|
||||
6. `idx_media_requests_active_tmdb` stays unchanged.
|
||||
|
||||
**Down:** re-add the three columns to `media_requests` and copy back the **1080p**
|
||||
target's external fields; drop `media_request_targets`; drop `is_anime`; collapse
|
||||
`request_integrations` to `kind`-PK by keeping each kind's `is_default` instance
|
||||
and discarding extras; drop the new columns/indexes.
|
||||
|
||||
**Caveat (explicit):** the down migration is **lossy** — rollback discards any
|
||||
additional instances (beyond one default per kind) and any 4K/anime target
|
||||
history. This is the unavoidable cost of collapsing a one-to-many back to
|
||||
one-to-one and is acceptable for a down migration.
|
||||
|
||||
**Go-side data access** (implementation, not SQL): `Integration` struct +
|
||||
`scanIntegration` gain the new fields; the `Store` interface gains target CRUD +
|
||||
`ListInstances(kind)`; `scanRequest` drops the external fields; `UpsertIntegration`
|
||||
becomes id-based CRUD with default-toggle handling in-transaction.
|
||||
|
||||
### 8. Testing
|
||||
|
||||
- **`routeTargets` — table-driven (core of the feature):** matrix of
|
||||
{1080p vs 2160p entitlement} × {`force_dual_quality` on/off} × {which defaults
|
||||
exist} × {anime vs not} → expected target set.
|
||||
- **`detectAnime`:** keyword `210024` present/absent; no name-substring false
|
||||
positives.
|
||||
- **Aggregate recompute:** all-complete, partial-failure (request stays active +
|
||||
`last_error`), all-failed (`outcome=failed`), mixed downloading/queued.
|
||||
- **Repository:** target CRUD; single-default-per-kind invariant rejects a second
|
||||
default; migration backfill yields exactly one 1080p target per previously
|
||||
submitted request.
|
||||
- **Adapter:** Sonarr `SubmitSeries` sets `seriesType=anime` only for anime
|
||||
targets on anime-enabled instances.
|
||||
- **Service:** extend `service_test.go` fakes — a 4K-entitled user's single
|
||||
request fans out to two adapter submissions with the right resolved instances;
|
||||
target-scoped retry re-submits only the failed target.
|
||||
- **Frontend:** instance-form default-invariant enforcement; queue renders
|
||||
per-target rows + retry.
|
||||
|
||||
## Risks & open considerations
|
||||
|
||||
- **Reconcile cost** scales with targets, not requests (≤2× today). Bounded and
|
||||
acceptable; the reconcile loop already batches.
|
||||
- **Anime keyword imperfection** is inherited from Seerr by design (parity
|
||||
choice). Mitigated by isolating `detectAnime` for future tuning.
|
||||
- **Client model drift**: removing top-level external fields from `Request` must
|
||||
be verified against `silo-android` / `silo-apple` deserialization before
|
||||
release (flagged in §6).
|
||||
|
||||
## References
|
||||
|
||||
- Seerr upstream: https://github.com/seerr-team/seerr
|
||||
(`server/api/themoviedb/constants.ts`, `server/entity/MediaRequest.ts`)
|
||||
- Overseerr anime discussions: #2876 (Sonarr for anime only), #3777 (configurable
|
||||
series type)
|
||||
- Existing code: `internal/requests/{service,repository,types,store}.go`,
|
||||
`internal/requests/{radarr,sonarr,arrclient}`, `internal/access/quality.go`,
|
||||
`migrations/139_media_requests.up.sql`
|
||||
@@ -0,0 +1,222 @@
|
||||
# Autoscan: Polling Radarr/Sonarr to Trigger Targeted Library Scans
|
||||
|
||||
**Status:** Design approved, ready for implementation plan
|
||||
**Date:** 2026-06-02
|
||||
**Area:** new `internal/autoscan`, `internal/taskmanager/tasks`, `migrations/`, `internal/api`, `web/src/pages/AdminRequests.tsx`
|
||||
|
||||
## Summary
|
||||
|
||||
Silo currently learns about new media via scheduled/manual library scans. When
|
||||
Radarr/Sonarr import a download, the new file only appears in Silo on the next
|
||||
scan. This adds an **autoscan poller**: a periodic background task that polls the
|
||||
configured Radarr/Sonarr instances for newly-imported files and enqueues
|
||||
**targeted folder scans** into Silo's existing scan queue, so requested (and
|
||||
non-requested) content shows up promptly.
|
||||
|
||||
The design follows the established autoscan pattern — poll arr import history,
|
||||
translate the imported paths, and trigger targeted scans — but kept deliberately
|
||||
simple. That pattern is normally built to orchestrate scans across many separate
|
||||
media-server boxes, which needs a bounded fan-out guard and a retry queue. Silo
|
||||
is a single service scanning *itself*, so the cross-node fan-out machinery is
|
||||
unnecessary — imported paths become folder jobs in Silo's own worker-bounded
|
||||
`scanqueue`.
|
||||
|
||||
## Goals
|
||||
|
||||
- Poll autoscan-enabled Radarr/Sonarr instances on an interval for import events.
|
||||
- Translate arr import paths into Silo media folders and enqueue targeted scans.
|
||||
- Reuse the Radarr/Sonarr instances already configured for requests
|
||||
(`request_integrations`) — no duplicate URLs/keys.
|
||||
- Coalesce bursts (a season's episodes) into minimal redundant scans.
|
||||
- Admin-configurable: global enable, interval, debounce, per-instance toggle,
|
||||
optional path rewrites, on-demand "poll now".
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No cross-node scan fan-out or bounded-concurrency semaphore (single service;
|
||||
the `scanqueue` workers are the bound).
|
||||
- No separate retry queue (a failed source poll simply does not advance its
|
||||
high-water mark, so the next cycle retries the window).
|
||||
- No arr webhook/"Connect" push integration (polling only, as requested; webhook
|
||||
push is a possible future add-on).
|
||||
- No separate autoscan source credential store — sources are existing
|
||||
`request_integrations`.
|
||||
|
||||
## Background
|
||||
|
||||
- **`internal/scantrigger`** already maps a filesystem path to a Silo
|
||||
`MediaFolder` and scan path: `Resolver.Resolve(Request{Path}) -> Target{Folder,
|
||||
Mode, Path}` (via `LongestMatchingRoot` / `MatchFolderForPath`).
|
||||
- **`internal/scanqueue`** executes scans through a worker-bounded queue:
|
||||
`Queuer.EnqueueScan(ctx, folderID, mode, path, trigger)` /
|
||||
`EnqueueScans(targets)`.
|
||||
- **`internal/requests/arrclient`** is a thin Radarr/Sonarr HTTP client
|
||||
(`New(baseURL, apiKey, httpClient)`, `GetJSON`, `DoJSON`) — reused for the new
|
||||
history call.
|
||||
- **`request_integrations`** holds each instance's `id`, `kind`, `base_url`, and
|
||||
Fernet-encrypted `api_key_ref`, resolved via the same secret resolver the
|
||||
request service uses.
|
||||
- The **task manager** runs periodic tasks via the `taskmanager.Task` interface
|
||||
with interval `DefaultTriggers()` (e.g. `ReconcileRequestsTask`).
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Data model & config
|
||||
|
||||
A new `internal/autoscan` package owns its own schema; `request_integrations`
|
||||
stays focused on fulfillment.
|
||||
|
||||
**`autoscan_settings`** (singleton, mirrors `request_settings`):
|
||||
|
||||
| column | type / default | purpose |
|
||||
|---|---|---|
|
||||
| `id` | bool PK default true, singleton CHECK | |
|
||||
| `enabled` | bool, default false | master on/off |
|
||||
| `poll_interval_minutes` | int, default 10 | poll task interval |
|
||||
| `debounce_seconds` | int, default 60 | folder re-scan suppression window |
|
||||
| `created_at`, `updated_at` | timestamptz | |
|
||||
|
||||
**`autoscan_sources`** (one row per autoscan-enabled instance):
|
||||
|
||||
| column | type | purpose |
|
||||
|---|---|---|
|
||||
| `integration_id` | text PK → `request_integrations(id)` ON DELETE CASCADE | the source instance |
|
||||
| `enabled` | bool, default false | per-instance autoscan toggle |
|
||||
| `path_rewrites` | jsonb, default `[]` | `[{from, to}]` prefix overrides |
|
||||
| `last_poll_at` | timestamptz, null | `history/since` high-water mark |
|
||||
| `created_at`, `updated_at` | timestamptz | |
|
||||
|
||||
**Reused, not duplicated:** `base_url` / `api_key_ref` come from
|
||||
`request_integrations`. Debounce/suppression state lives in **Redis**
|
||||
(`autoscan:scanned:{folder_id}`, TTL = `debounce_seconds`), not Postgres.
|
||||
|
||||
### 2. Poll cycle
|
||||
|
||||
`autoscan.Service.PollOnce(ctx)` (invoked by the periodic task):
|
||||
|
||||
1. Load `autoscan_settings`; if `!enabled`, return immediately (no-op).
|
||||
2. Load `autoscan_sources WHERE enabled`, joined to `request_integrations` for
|
||||
`kind` / `base_url` / `api_key_ref`.
|
||||
3. For each source (sequential — a handful of instances):
|
||||
1. Decrypt the API key via the secret resolver.
|
||||
2. Record `cycleStart := now`.
|
||||
3. `arrclient.GetJSON("/api/v3/history/since?date={last_poll_at}&eventType=downloadFolderImported")`
|
||||
— both Radarr and Sonarr expose this. `last_poll_at` null → a sane floor
|
||||
(do not scan all history on first enable; floor at `cycleStart`).
|
||||
4. Extract the imported **file path** per event (Radarr: movie file path;
|
||||
Sonarr: episode file `outputPath`); take its **parent directory**.
|
||||
5. On success, advance that source's `last_poll_at = cycleStart`. On failure
|
||||
(arr unreachable, decode error), log and **do not** advance — next cycle
|
||||
retries the window.
|
||||
4. For each extracted directory: apply path rewrites (§3), resolve via
|
||||
`scantrigger.Resolver` → `Target{Folder, Mode, Path}`. Unresolvable paths
|
||||
(outside Silo's media folders) are logged and skipped.
|
||||
5. **Dedupe to unique folders** for this cycle; for each, check the Redis
|
||||
suppression key — if absent, `EnqueueScan(..., trigger="autoscan")` and set
|
||||
the key with TTL `debounce_seconds`; if present, skip (already scanned this
|
||||
window).
|
||||
|
||||
**Debounce rationale:** because the poller batches a whole interval of imports
|
||||
per `history/since` call, within-cycle folder dedup does most of the work. The
|
||||
Redis suppression key only prevents re-enqueuing the same folder on back-to-back
|
||||
cycles. Silo's scanner is idempotent, so a season importing across two cycles
|
||||
simply gets a second (cheap) scan that picks up the rest. This is simpler than a
|
||||
"collect-expired" debounce plus a retry queue, which the polling + idempotent
|
||||
re-scan model makes unnecessary.
|
||||
|
||||
### 3. Path resolution & scan enqueue
|
||||
|
||||
A small pipeline in front of the existing scan plumbing — no new scan logic:
|
||||
|
||||
1. **Rewrite** (per source, optional): apply the first matching `path_rewrites`
|
||||
entry as a prefix replacement (e.g. `/data/media -> /mnt/media`). No rewrites
|
||||
→ path used as-is (shared-mount common case).
|
||||
2. **Resolve** via `scantrigger.Resolver.Resolve(Request{Path: rewrittenDir})`
|
||||
→ `Target{Folder, Mode, Path}`. No matching media folder → log + skip.
|
||||
3. **Enqueue** deduped targets via `scanqueue` `EnqueueScans` with
|
||||
`trigger="autoscan"` (attributable in scan history, flows through the same
|
||||
worker-bounded queue).
|
||||
|
||||
### 4. Scheduling & concurrency
|
||||
|
||||
- **`internal/taskmanager/tasks/autoscan_poll.go`** implements `taskmanager.Task`
|
||||
(`Key: "autoscan_poll"`, name/description/category, `DefaultTriggers` = interval
|
||||
trigger seeded from `poll_interval_minutes`). Its run body calls
|
||||
`autoscan.Service.PollOnce(ctx)`.
|
||||
- **Disabled = cheap no-op:** the task stays registered; `PollOnce` returns early
|
||||
when `enabled` is false. Toggling autoscan never (de)registers tasks.
|
||||
- **Interval changes:** updating `poll_interval_minutes` reconfigures the task's
|
||||
interval trigger via the task manager's existing trigger-config path; if a live
|
||||
update is unavailable it applies on next start.
|
||||
- **No overlap, no semaphore:** the task manager serializes a task's executions;
|
||||
sources are polled sequentially; scan execution is bounded by `scanqueue`. The
|
||||
single-service simplification: no bounded-fan-out semaphore is needed.
|
||||
- **Primary/integrated server only** (same as the request reconcile task), not on
|
||||
standalone transcode/proxy worker nodes.
|
||||
|
||||
### 5. Admin API & UI
|
||||
|
||||
**API** (admin-gated, `/api/v1/admin/autoscan`, mirroring the request-settings
|
||||
handlers; never echoes API keys):
|
||||
|
||||
- `GET/PUT /autoscan/settings` — `{enabled, poll_interval_minutes, debounce_seconds}`
|
||||
- `GET /autoscan/sources` — per autoscan-eligible instance (joined from
|
||||
`request_integrations`): `{integration_id, name, kind, enabled, path_rewrites,
|
||||
last_poll_at}`
|
||||
- `PUT /autoscan/sources/{integration_id}` — set `{enabled, path_rewrites}`
|
||||
- `POST /autoscan/trigger` — run `PollOnce` once on demand; tight per-admin rate
|
||||
limit so it cannot hammer the arr instances
|
||||
- `GET /autoscan/status` — `{enabled, per-source last_poll_at, last scan summary}`
|
||||
|
||||
**UI:** a new **"Autoscan" tab** in `web/src/pages/AdminRequests.tsx` (alongside
|
||||
Queue / Settings / Integrations / User Overrides):
|
||||
|
||||
- Global controls: enable switch, poll interval, debounce window.
|
||||
- Per-source list — one row per Radarr/Sonarr instance with an autoscan toggle,
|
||||
an optional `from → to` path-rewrite editor, and a read-only "last polled".
|
||||
- A **"Poll now"** button → `POST /autoscan/trigger`.
|
||||
|
||||
Rate limits and admin-auth follow Silo's existing request-admin conventions.
|
||||
|
||||
### 6. Testing
|
||||
|
||||
- **Path rewrite:** prefix replacement, first-match ordering, no-match
|
||||
passthrough, trailing-slash handling.
|
||||
- **arr history client:** table-driven `httptest` fixtures for both Radarr and
|
||||
Sonarr import-event JSON shapes; asserts the correct file path is extracted and
|
||||
its parent folder taken.
|
||||
- **Dedupe + suppression:** many paths in one folder → one target; folder with a
|
||||
live suppression key skipped; absent/expired key enqueues.
|
||||
- **`PollOnce` (integration seam):** fakes for source store, arr history client,
|
||||
`scantrigger.Resolver`, `scanqueue.Queuer`, Redis — asserts deduped folder
|
||||
scans enqueued with `trigger="autoscan"`, `last_poll_at` advanced on success,
|
||||
**not** advanced on per-source failure, clean no-op when disabled.
|
||||
- **Unresolvable path:** logged + skipped, never enqueued or fatal.
|
||||
- **Repository / migration:** `autoscan_settings` singleton + `autoscan_sources`
|
||||
CRUD and the `request_integrations` join; migration applies + rolls back
|
||||
(sidecar table, `ON DELETE CASCADE`).
|
||||
- **Frontend:** Autoscan tab renders, per-source toggle / path-rewrite editor
|
||||
round-trips, "Poll now" calls the trigger endpoint.
|
||||
|
||||
## Risks & open considerations
|
||||
|
||||
- **First-enable floor:** `last_poll_at` null must floor at "now" (not epoch) so
|
||||
enabling autoscan doesn't enqueue a scan for every historical import.
|
||||
- **history/since shape differences:** Radarr vs Sonarr import-event payloads
|
||||
differ; the path extraction is per-kind and covered by fixtures.
|
||||
- **Path rewrites vs scantrigger roots:** if neither rewrites nor Silo's media
|
||||
folders match the arr path, the import is silently skipped (logged) — this is
|
||||
the expected "content outside Silo's libraries" behavior, but a misconfigured
|
||||
rewrite will look like "autoscan does nothing"; the `last_poll_at` + status
|
||||
endpoint and skip logs are the debugging surface.
|
||||
- **Idempotent re-scan:** relies on Silo's scanner being safe to run repeatedly
|
||||
on a folder (it is — re-scan finds new files).
|
||||
|
||||
## References
|
||||
|
||||
- Existing Silo plumbing reused by this design: `internal/scantrigger/scantrigger.go`,
|
||||
`internal/scanqueue/`, `internal/requests/arrclient/`,
|
||||
`internal/taskmanager/tasks/reconcile_requests.go`
|
||||
- Prior art: the general autoscan pattern (poll arr import history → translate
|
||||
paths → trigger targeted scans). Silo's variant drops the cross-node fan-out
|
||||
guard and retry queue, which only a multi-box orchestrator needs.
|
||||
@@ -0,0 +1,262 @@
|
||||
# Autoscan as a Pluggable Scan-Source Category — Design Spec
|
||||
|
||||
**Date:** 2026-06-02
|
||||
**Status:** Approved design, pre-implementation
|
||||
**Goal:** Turn autoscan from a Requests-coupled, arr-only feature into a standalone **Autoscan** category whose change-detection providers are out-of-process Silo plugins, with Sonarr/Radarr as the first provider.
|
||||
|
||||
**Architecture (one sentence):** Silo's host keeps a thin, provider-agnostic engine that resolves changed paths to library folders and enqueues rescans; each *way of noticing change* (arr, later inotify/ceph) is an installable plugin that Silo pulls on a timer through a new additive `scan_source.v1` capability.
|
||||
|
||||
**Tech stack:** Go (silo-server host + plugins), `silo-plugin-sdk` (protobuf capability contract, consumed as a tagged module), existing `internal/plugins` + `internal/pluginhost` runtime, existing `scantrigger`/`scanqueue` scan pipeline, React/TS admin UI.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## 1. Background & motivation
|
||||
|
||||
The current autoscan implementation (delivered in the now-closed PR #43, still on local `main`) is in-process Go that:
|
||||
|
||||
- polls Sonarr/Radarr `/api/v3/history/since` on a host timer,
|
||||
- rewrites arr-side paths to Silo paths, dedups to parent dirs,
|
||||
- resolves each to a `MediaFolder` and enqueues a targeted scan,
|
||||
- is configured against `request_integrations` rows (hard FK, `ON DELETE CASCADE`), living as a tab inside the **Requests** admin page.
|
||||
|
||||
Two structural problems:
|
||||
|
||||
1. **It is arr-only and not extensible.** Adding inotify, Ceph notifications, or any other change source means more bespoke host code.
|
||||
2. **It is subordinate to Requests.** A source cannot exist without a Requests integration row; deleting the row destroys the source.
|
||||
|
||||
Silo already ships a full out-of-process plugin runtime (`internal/plugins`, `internal/pluginhost`): manifests, an installer, typed capability descriptors (`metadata_provider.v1`, `request_router.v1`, `media_analyzer.v1`, `scheduled_task.v1`, `event_consumer.v1`, …), a host-driven scheduled-task mechanism, and a rate-limited plugin→host event path. This design uses that runtime to make change-detection pluggable.
|
||||
|
||||
## 2. Goals / non-goals
|
||||
|
||||
**Goals**
|
||||
|
||||
- A new **Autoscan** admin category, independent of Requests.
|
||||
- A new additive `scan_source.v1` plugin capability — **breaks no existing plugin**.
|
||||
- A **client-pull** contract: Silo asks each source "what changed since `<marker>`?" on a timer.
|
||||
- Sonarr/Radarr re-implemented as the first scan-source plugin; the closed PR's arr logic relocates into it.
|
||||
- A provider-agnostic host engine reusable by every future scan source.
|
||||
- Optional reuse of Requests' arr connection details, *or* standalone connection entry.
|
||||
|
||||
**Non-goals (this design)**
|
||||
|
||||
- inotify / Ceph / other providers (the contract must *accommodate* them; we do not build them).
|
||||
- A push (plugin-initiated) delivery mode. Pull only; push sources buffer internally and answer on pull (see §6).
|
||||
- Refactoring the Requests subsystem itself.
|
||||
|
||||
## 3. Architecture overview
|
||||
|
||||
```
|
||||
┌──────────────────────────── Silo (host) ────────────────────────────┐
|
||||
│ Autoscan category (new; not under Requests) │
|
||||
│ • Sources: which scan_source plugin instance, on/off, interval │
|
||||
│ • Connections: reuse a Requests arr server, or enter own │
|
||||
│ • Per-source state: opaque marker, last-run status │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ AutoscanStore — generic engine (provider-agnostic) │
|
||||
│ per tick, per enabled source: │
|
||||
│ call plugin PollChanges(marker) │
|
||||
│ → resolve each Silo-native path → MediaFolder │
|
||||
│ → suppress duplicates → enqueue scan (existing scanner) │
|
||||
│ → store returned next_marker │
|
||||
│ │ scan_source.v1 (pull) │
|
||||
└──────────────────────────────┼───────────────────────────────────────┘
|
||||
▼
|
||||
┌──────── Sonarr/Radarr scan-source plugin ────────┐
|
||||
│ (new installable repo, like silo-plugin-tmdb) │
|
||||
│ receives resolved arr address + key │
|
||||
│ polls /history (imports + renames) │
|
||||
│ returns RAW source paths + next marker │
|
||||
│ (host applies the source's rewrites) │
|
||||
└──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Five components**
|
||||
|
||||
1. **`scan_source.v1`** (in `silo-plugin-sdk`) — the pull contract. §5.
|
||||
2. **AutoscanStore** (host) — generic resolve→suppress→enqueue→marker engine. §7.
|
||||
3. **Connection resolution** (host) — reference-or-inline → concrete credentials. §8.
|
||||
4. **Autoscan category + data model** (host) — sources, connections, decoupled from Requests. §9.
|
||||
5. **arr scan-source plugin** (new repo) — first provider. §10.
|
||||
|
||||
## 4. Responsibility split (the dividing line)
|
||||
|
||||
| Concern | Owner | Rationale |
|
||||
|---|---|---|
|
||||
| Talk to arr; poll `/history`; imports + renames; bounded window; first-run | **Plugin** | Provider-specific I/O. |
|
||||
| **Path rewrites** (per-source config + application) | **Host** | _(Revised post-merge: the SDK landed host-owned rewrites.)_ The plugin returns RAW source-namespace paths in `source_paths`; the host applies each source's rewrite rules (longest-`From`-first) before resolving. Keeps every provider dumb and rewrite behavior uniform. Rewrites are edited per-source in the Autoscan UI and stored in `autoscan_sources.path_rewrites`. |
|
||||
| Resolve Silo-native path → `MediaFolder`; dedupe; **enqueue rescan** | **Host** | Touches Silo internals (scan queue, DB, library map) that plugins cannot and must not reach. |
|
||||
| Opaque per-source marker; poll interval; enabled; last-run status | **Host** | Scheduling and bookkeeping are the engine's job. |
|
||||
| Connection selection (reuse-from-Requests vs own) and credential resolution | **Host** | Only the host knows about Requests; the plugin receives resolved creds. |
|
||||
|
||||
The plugin's responsibility ends at **"here are the changed paths, in Silo's terms."** Everything past that is the host engine, which is provider-agnostic.
|
||||
|
||||
## 5. The `scan_source.v1` contract (centerpiece)
|
||||
|
||||
A new, additive capability in `silo-plugin-sdk`. Existing plugins never declare it and are unaffected (§11).
|
||||
|
||||
### Protobuf (as implemented in silo-plugin-sdk)
|
||||
|
||||
```proto
|
||||
// scan_source.v1 — host pulls changed paths from a provider on a timer.
|
||||
service ScanSource {
|
||||
// Host calls this on each poll tick for a configured scan_source capability.
|
||||
rpc PollChanges(PollChangesRequest) returns (PollChangesResponse);
|
||||
}
|
||||
|
||||
message PollChangesRequest {
|
||||
string capability_id = 1; // which configured scan_source instance
|
||||
string marker = 2; // opaque; empty string on first run
|
||||
ResolvedConnection connection = 3; // resolved upstream credentials (see §8)
|
||||
}
|
||||
|
||||
// Concrete upstream credentials the host resolves and hands to the plugin each
|
||||
// poll, so the plugin stays credential-source-agnostic and stores no secrets.
|
||||
message ResolvedConnection {
|
||||
string base_url = 1;
|
||||
string api_key = 2;
|
||||
}
|
||||
|
||||
message PollChangesResponse {
|
||||
// Absolute changed paths in the provider/source namespace (RAW — NOT yet
|
||||
// rewritten). Files or directories. The host applies the source's rewrite
|
||||
// rules and validates before enqueueing.
|
||||
repeated string source_paths = 1;
|
||||
// Opaque continuation token. Host stores verbatim and echoes it back
|
||||
// on the next PollChanges. The plugin never assumes the host parses it.
|
||||
string next_marker = 2;
|
||||
}
|
||||
```
|
||||
|
||||
### Contract rules
|
||||
|
||||
- **Pull only.** The host owns the timer and calls `PollChanges`. The plugin never initiates.
|
||||
- **Opaque marker.** The host stores and echoes `next_marker` without interpreting it. arr uses an RFC3339 timestamp; a future provider may use a sequence number or event id — no host change required.
|
||||
- **First run.** `marker == ""` ⇒ the plugin starts from "now" (do not replay full history).
|
||||
- **Idempotent, at-least-once.** The host may re-issue the same `marker` if it failed to persist `next_marker`; providers must tolerate returning overlapping paths. The host's suppression layer (§7) absorbs duplicates.
|
||||
- **Error semantics.** A failed `PollChanges` (e.g. arr unreachable) ⇒ the host keeps the *old* marker and retries next tick; nothing is skipped or advanced.
|
||||
- **Raw source paths.** `source_paths` are in the provider's namespace, NOT rewritten. The host applies each source's path-rewrite rules (host-owned, per-source) before resolving to a library folder.
|
||||
- **Credentials per call.** The host resolves the source's connection (§8) and passes the concrete `{base_url, api_key}` in `PollChangesRequest.connection` on every poll. The plugin reads them from the request — it stores no credentials and never knows whether they came from its own config or a reused Requests link. Delivered over the local host↔plugin gRPC channel only.
|
||||
|
||||
The capability needs **only** `PollChanges` for v1. A `TestConnection`/validate RPC is explicitly deferred (YAGNI; connection validity surfaces as a failed poll).
|
||||
|
||||
## 6. Marker & delivery semantics
|
||||
|
||||
- One opaque marker stored per source, host-side.
|
||||
- Pull-only does **not** exclude push-style providers: an inotify/Ceph plugin maintains its own internal buffer of observed changes and flushes it (and advances its marker) on each `PollChanges`. Cost: up to one poll-interval of latency. This keeps a single uniform contract.
|
||||
|
||||
## 7. Host: AutoscanStore engine
|
||||
|
||||
Provider-agnostic. Per enabled source, per tick:
|
||||
|
||||
1. `paths, next := plugin.PollChanges(marker)`; on error, log + keep marker + continue.
|
||||
2. For each `dir` in `uniqueParentDirs(paths)`:
|
||||
- `target := scantrigger.Resolve(dir)`. Unresolvable (outside Silo's media folders) ⇒ quiet skip.
|
||||
- Suppression key `fmt.Sprintf("%d|%s", folderID, target.Path)` with debounce TTL; already-claimed ⇒ skip.
|
||||
- Accumulate `target`; record claimed key.
|
||||
3. `scanqueue.EnqueueScans(targets)`. On failure, release claims and **do not** advance the marker.
|
||||
4. On success, persist `next_marker`.
|
||||
|
||||
This is the salvaged core of PR #43 (`internal/autoscan/service.go` resolve/suppress/enqueue, `dedupe.go`, `suppress.go`), generalized so its input is "paths from a provider" rather than "paths from arr." Path-rewrite helpers (`rewrite.go`, `suggest.go`, `history.go`, the arr history client) leave the host and move into the plugin (§10).
|
||||
|
||||
## 8. Host: Connections (reuse or own)
|
||||
|
||||
A **connection** is "an arr server Silo can reach," created two ways:
|
||||
|
||||
- **Own:** name + base URL + API key, entered in the Autoscan category; the key is Fernet-encrypted at rest (same mechanism as existing service credentials — never plaintext in JSONB).
|
||||
- **Linked (reuse):** a live reference to an existing Requests `request_integrations` arr server. Resolved at use-time, so edits in Requests propagate.
|
||||
|
||||
**Resolution:** on each poll the host resolves the chosen connection (own or linked) into concrete `{base_url, api_key}` and passes them in `PollChangesRequest.connection` (§5). The plugin is credential-source-agnostic — it only ever sees resolved values, fresh each poll, so a change to a reused Requests credential propagates immediately with no re-push. This "reuse or own" behaviour is therefore free for every future provider. (Resolving per-call rather than caching in plugin config also means no decrypted secret is ever stored at rest in the plugin.)
|
||||
|
||||
**Decoupling from Requests:** the link is **soft and optional** — `SET NULL` / surfaced-as-"needs attention", never cascading. Requests has no knowledge of Autoscan. Autoscan and Requests are peers; Autoscan merely *offers* to borrow a Requests connection.
|
||||
|
||||
## 9. Host: Autoscan category & data model
|
||||
|
||||
New top-level **Autoscan** admin section (its own nav entry, not a Requests tab). Manages **Connections** (§8) and **Sources**.
|
||||
|
||||
A **source** corresponds to one configured `scan_source.v1` capability instance and carries:
|
||||
|
||||
- **Host-owned per-source config:** path rewrites (`autoscan_sources.path_rewrites`, edited in the Autoscan Sources UI) and any provider-specific settings. _(Revised post-merge: rewrites moved host-side; the plugin has no config.)_
|
||||
- **Host-owned per-source state:** the chosen connection, enabled on/off, poll interval (per-source, not one global timer), the opaque marker, and last-run status/error.
|
||||
|
||||
**Install flow + source auto-discovery (resolves §14 risk #1).** A scan-source plugin (Sonarr/Radarr, etc.) is installed through the **normal `/admin/plugins`** page — there is no separate install path; the installer doesn't special-case the capability type. Source rows are then never created by hand: the host enumerates every installed `scan_source.v1` capability (via the plugin installation store) and *seeds a disabled, connection-less row* per capability (`INSERT … ON CONFLICT (installation_id, capability_id) DO NOTHING`, never disturbing existing config). Discovery runs **both on each poll cycle AND when the Autoscan sources list is viewed** (so a freshly-installed plugin appears in Autoscan immediately, even when autoscan is disabled and nothing is polling). The operator then binds a connection and enables it. Consequently **`connection_id` is nullable**: a freshly-discovered source has none, and the engine skips any enabled source with no connection (recording a "no connection bound" status). The admin API rejects enabling a source that has no effective connection.
|
||||
|
||||
**Per-source interval** is honoured as a floor: the global poll task runs at `default_poll_interval_seconds`, and within a cycle the engine skips a source whose `last_run_at` is newer than its own `poll_interval_seconds` (falling back to the default). A source therefore polls *at most* every N seconds, never more often than the global cadence.
|
||||
|
||||
### Schema changes (host)
|
||||
|
||||
Rework the PR #43 tables; **no FK into `request_integrations`**:
|
||||
|
||||
- `autoscan_connections` — `id`, `name`, `kind` (e.g. `sonarr`/`radarr`), and *either* own credentials (`base_url`, `api_key_ref` encrypted) *or* a nullable soft link `request_integration_id` (`ON DELETE SET NULL`).
|
||||
- `autoscan_sources` — `id`, `(installation_id, capability_id)` (unique; the discovered identity), **nullable** `connection_id → autoscan_connections` (`ON DELETE RESTRICT`; null until the operator binds one), `enabled`, `poll_interval_seconds` (nullable; null = use default), `marker` (opaque text, nullable), `last_run_at`, `last_error`. Creation-time validity for connections (own creds OR a link) is enforced at the application layer (the DB CHECK would conflict with `ON DELETE SET NULL` on the reuse link).
|
||||
- `autoscan_settings` (optional) — retain a global enable + default interval; per-source interval overrides it.
|
||||
|
||||
A migration supersedes PR #43's `171_autoscan` schema (which is not yet upstream, so this is a forward redefinition, not a production migration of live data on `origin/main`).
|
||||
|
||||
## 10. Plugin: Sonarr/Radarr scan-source
|
||||
|
||||
A new installable repo structured like `silo-plugin-tmdb` (standalone Go module, `manifest.json` declaring `scan_source.v1`, gRPC `main.go`, depends on the new tagged `silo-plugin-sdk`). Receives resolved arr `{base_url, api_key}` as config. Implements `PollChanges`:
|
||||
|
||||
1. Treat `marker` as the "since" timestamp (empty ⇒ now).
|
||||
2. Call arr `/api/v3/history/since`, capped by a bounded window (24h max-lookback floor + overlap buffer — relocated from PR #43) so a long outage cannot pull an oversized response.
|
||||
3. Extract paths: **imports** (`downloadFolderImported.importedPath`) and **renames** (`episodeFileRenamed`/`movieFileRenamed`: both new `path` and old `sourcePath`). Deletes ignored (upgrade-deletes are covered by the paired import; standalone deletes carry no path).
|
||||
4. Return `{source_paths = the RAW arr paths, next_marker = newest history timestamp}`. The plugin does NOT rewrite — the host normalizes separators and applies the source's rewrite rules. _(Revised post-merge.)_
|
||||
|
||||
**Rewrite config + "Sync from arr":** the rewrite list and the suffix-match suggester (which matches arr root folders to Silo media folders) live in the plugin. The suggester needs Silo's media-folder list, obtained via the existing host library-listing service (`internal/pluginhost` library lister) exposed to plugins. The relocated logic is PR #43's `suggest.go`/`suggest_deps.go`/`rewrite.go`/`history.go`.
|
||||
|
||||
## 11. Backward compatibility (hard requirement)
|
||||
|
||||
Adding `scan_source.v1` **must not break existing plugins.** It doesn't, by construction:
|
||||
|
||||
- Capabilities are independent opt-in declarations. Each host subsystem **filters for its own type and ignores the rest** (verified: `metadata_provider.v1`, `scheduled_task.v1`, `event_consumer.v1`, … each use `if type != mine { skip }`). There is **no global allowlist that rejects unknown capability types.**
|
||||
- **Additive-only rule:** the SDK change *adds* new proto messages/service + a new capability string. It never modifies the shape of an existing capability. (Mutating an existing capability is the only thing that would break plugins; we forbid it.)
|
||||
- **No forced upgrades:** existing plugin repos stay pinned to their current SDK version and build unchanged. Only a plugin that *wants* to be a scan source adopts the new SDK version.
|
||||
|
||||
## 12. Build order & repo decomposition
|
||||
|
||||
One shared contract, three repos (+ catalog), in dependency order:
|
||||
|
||||
1. **`silo-plugin-sdk`** (long pole): define `scan_source.v1` (proto + generated code + manifest helper + runtime server scaffold), tag a new version (e.g. `v0.5.0`). The tagged release is a hard serialization point — host and plugin cannot build against the contract until it exists.
|
||||
2. **silo-server (host)** — bump SDK dep; build AutoscanStore (generalized), the `scan_source.v1` caller (timer → `PollChanges`), connections + resolution, the Autoscan category UI, retire the in-process arr autoscan, add the reworked migration. *Parallelizable with (3) once (1) is tagged.*
|
||||
3. **arr plugin (new repo)** — depends on SDK `v0.5.0`; implement `PollChanges`, rewrites, "Sync from arr", bounded window.
|
||||
4. **`silo-plugins`** (catalog) — add the entry so the arr plugin is installable.
|
||||
|
||||
Each repo gets its **own implementation plan** (separate `writing-plans` pass); this single spec defines the shared contract and per-side responsibilities they all build against.
|
||||
|
||||
**Local logistics:** only `silo-server` and `silo-plugin-tmdb` are currently checked out. Implementation will require `silo-plugin-sdk` and a new arr-plugin repo (and later `silo-plugins`) available locally.
|
||||
|
||||
## 13. Salvage map (from closed PR #43)
|
||||
|
||||
| PR #43 code | Destination |
|
||||
|---|---|
|
||||
| arr `/history` client, imports + renames, bounded window (`history.go`) | arr plugin |
|
||||
| path rewrite apply (`rewrite.go`) + "Sync from arr" suggester (`suggest.go`, `suggest_deps.go`) | arr plugin |
|
||||
| `last_poll_at` cursor | host, generalized to opaque per-source `marker` |
|
||||
| resolve + dedupe + suppress + enqueue (`service.go`, `dedupe.go`, `suppress.go`) | host AutoscanStore (generic) |
|
||||
| `171_autoscan` schema (FK → `request_integrations`) | host, reworked: `autoscan_connections` + `autoscan_sources`, no requests FK |
|
||||
| Autoscan admin tab under Requests (`AdminRequests.tsx`) | host, as its own Autoscan category |
|
||||
|
||||
## 14. Risks & open questions
|
||||
|
||||
1. **Plugin multiplicity for multiple arr servers.** Does the runtime support multiple configured instances of one plugin (one per arr server), or is config per-installation only? If the latter, "two Sonarrs + one Radarr" implies either multiple installations or a plugin that manages several connections internally. *Resolve before the host/plugin plans by reading `internal/plugins` installation + capability-instance handling.*
|
||||
2. **Plugin config UI richness.** The rewrite table + a "Sync from arr" action are richer than typical key-value plugin config. Verify the plugin config-schema/admin-config UI can host a structured list and an action button; if not, design a minimal affordance (e.g. suggester runs on first configure, or a dedicated plugin HTTP route via `http_routes.v1`).
|
||||
3. **Library-folder list exposure to plugins.** Confirm the `internal/pluginhost` library lister is reachable by a `scan_source` plugin for the suggester. If not, add a host service method.
|
||||
4. **Marker persistence vs at-least-once.** Ensure marker write happens only after successful enqueue, matching PR #43's "don't advance on failure" guarantee.
|
||||
5. **Closed PR disposition.** The in-process autoscan on local `main` (34 commits) should be removed/replaced rather than shipped; sequence the host plan to retire it cleanly.
|
||||
|
||||
## 15. Testing strategy
|
||||
|
||||
- **SDK:** proto compiles; manifest helper round-trips `scan_source.v1`; runtime scaffold serves `PollChanges`.
|
||||
- **Host engine:** table-driven tests for resolve→suppress→enqueue→marker (port PR #43's `service_test.go`, generalized to a fake `ScanSource` client): dedupe, distinct-paths-same-folder, disabled no-op, poll failure keeps marker, enqueue failure releases claims + keeps marker, opaque-marker pass-through. Run in the libvips-equipped environment (host handler tests link libvips via CGO — see workspace note).
|
||||
- **Connections:** resolution of own vs linked; linked falls back gracefully when the Requests row is deleted (`SET NULL`).
|
||||
- **arr plugin:** history parsing (imports + renames, ignores unrelated events), rewrite application, bounded window, first-run "from now"; suffix-match suggester cases (unique/ambiguous/covered/no-op/normalization) ported from PR #43.
|
||||
- **Compat:** an existing-style plugin manifest with no `scan_source.v1` is unaffected by host scan-source dispatch.
|
||||
|
||||
## 16. Out of scope / future
|
||||
|
||||
- inotify and Ceph-notify scan-source plugins (the contract accommodates them via the buffer-and-flush pattern of §6).
|
||||
- A push/event delivery mode, if pull latency ever proves insufficient.
|
||||
- A `TestConnection` RPC on the capability.
|
||||
- Migrating Requests onto a shared connection abstraction (today's reuse is a one-directional soft link only).
|
||||
@@ -0,0 +1,173 @@
|
||||
# Autoscan: Sync Path Rewrites from Arr Root Folders
|
||||
|
||||
**Status:** Design approved, ready for implementation plan
|
||||
**Date:** 2026-06-02
|
||||
**Area:** `internal/autoscan`, `internal/api`, `web/src/pages/AdminRequests.tsx`
|
||||
**Extends:** the autoscan feature (`docs/superpowers/specs/2026-06-02-autoscan-arr-polling-design.md`)
|
||||
|
||||
## Summary
|
||||
|
||||
Autoscan resolves an imported file path against Silo's media folders, but
|
||||
Radarr/Sonarr often report files under paths that differ from how Silo mounts
|
||||
the same content — so operators must hand-author per-root `path_rewrites`. For a
|
||||
multi-instance, many-root deployment this can be dozens of rows.
|
||||
|
||||
This adds a **"Sync rewrites from arr"** convenience: for one instance, query its
|
||||
root folders, match each root to a Silo media folder by **shared trailing path
|
||||
segments**, and present the resulting `{from, to}` rewrites as a **preview** the
|
||||
admin reviews and commits. Manual rewrites remain the source of truth; sync only
|
||||
*fills the editor* — nothing persists until the admin saves.
|
||||
|
||||
The matcher is purely structural (no assumptions about any particular path
|
||||
layout) so it helps any deployment whose arr and Silo share folder names at some
|
||||
depth, and degrades gracefully (no shared names → "unmatched" → manual entry)
|
||||
when they do not.
|
||||
|
||||
## Goals
|
||||
|
||||
- One-click suggestion of `path_rewrites` for an instance from its arr root folders.
|
||||
- Correct disambiguation when multiple Silo folders share a leaf name.
|
||||
- Preview-then-commit: the admin sees proposed/unmatched/ambiguous and saves
|
||||
through the existing per-source save. Manual rows are never removed or overwritten.
|
||||
- Deployment-agnostic: no hard-coded paths; works for any structure; surfaces
|
||||
match confidence so weak matches can be rejected.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No automatic/background writing of rewrites — the admin always commits.
|
||||
- No new persistence path — apply is the existing `PUT /sources/{id}`.
|
||||
- No case-insensitive matching (case-sensitive segments; correct on Linux). A
|
||||
toggle can be added later if needed.
|
||||
- No "sync all instances at once" — per-instance, matching the editor UI.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Matching function (core, pure)
|
||||
|
||||
A single side-effect-free function in `internal/autoscan`:
|
||||
|
||||
```go
|
||||
func suggestRewrites(arrRoots []string, siloFolderPaths []string, existing []PathRewrite) RewriteSuggestions
|
||||
```
|
||||
|
||||
```go
|
||||
type RewriteSuggestions struct {
|
||||
Proposed []ProposedRewrite `json:"proposed"`
|
||||
Unmatched []string `json:"unmatched"`
|
||||
Ambiguous []AmbiguousRoot `json:"ambiguous"`
|
||||
Covered []string `json:"covered"`
|
||||
}
|
||||
type ProposedRewrite struct {
|
||||
From string `json:"from"` // arr root
|
||||
To string `json:"to"` // matched Silo folder path
|
||||
MatchDepth int `json:"match_depth"` // trailing segments shared (confidence)
|
||||
}
|
||||
type AmbiguousRoot struct {
|
||||
Root string `json:"root"`
|
||||
Candidates []string `json:"candidates"`
|
||||
}
|
||||
```
|
||||
|
||||
**Normalization** (applied to every path before comparison): backslashes → `/`
|
||||
(reuse `normalizeSeparators`), strip a trailing `/`, collapse duplicate slashes.
|
||||
This makes any path format (POSIX, Windows, single- or many-root) comparable.
|
||||
|
||||
**Per arr root:**
|
||||
1. **Covered-first:** if any `existing` rewrite's `From` already matches the root
|
||||
(exact, or root starts with `From + "/"`, using the same boundary rule as
|
||||
`applyRewrites`), classify as **Covered** and skip — so sync never re-proposes
|
||||
what a manual or broad rule already handles.
|
||||
2. Split both the root and each Silo folder path into segments; compute the
|
||||
**longest common trailing-segment suffix** (full-segment equality, so `4ktv7`
|
||||
does not match `4ktv70`).
|
||||
3. Among Silo folders, take the maximum suffix length `> 0` and the folders that
|
||||
achieve it:
|
||||
- exactly one → **Proposed** `{From: root, To: folder, MatchDepth: n}`
|
||||
- more than one → **Ambiguous** `{root, candidates}`
|
||||
- zero → **Unmatched**
|
||||
|
||||
No deployment constants appear in the function; it operates only on the string
|
||||
slices it is given.
|
||||
|
||||
### 2. Preview endpoint & service
|
||||
|
||||
**Endpoint:** `GET /api/v1/admin/autoscan/sources/{id}/rewrite-suggestions` →
|
||||
`RewriteSuggestions` JSON (admin-gated, like the other autoscan routes).
|
||||
|
||||
**Service method** `Service.SuggestRewrites(ctx, integrationID) (RewriteSuggestions, error)`:
|
||||
1. `Repository.GetSource(id)` — a new single-source variant of the existing join
|
||||
that returns `kind`/`base_url`/`api_key_ref`/`path_rewrites` even when the
|
||||
source is disabled. Not found → `ErrIntegrationNotFound`.
|
||||
2. Decrypt the API key via the existing `SecretResolver`.
|
||||
3. `RootFolderClient.RootFolders(ctx, baseURL, apiKey)` → arr root paths.
|
||||
4. `FolderLister.ListFolderPaths(ctx)` → all Silo media-folder paths.
|
||||
5. Return `suggestRewrites(arrRoots, siloPaths, source.PathRewrites)`.
|
||||
|
||||
**Two new narrow `Service` dependencies** (injected like `history`/`resolver`):
|
||||
- `RootFolderClient.RootFolders(ctx, baseURL, apiKey string) ([]string, error)` —
|
||||
an `arrclient`-backed impl reusing `arrclient.ListRootFolders` (returns paths only).
|
||||
- `FolderLister.ListFolderPaths(ctx) ([]string, error)` — a thin adapter over
|
||||
`catalog.FolderRepository.List()` flattening each folder's `Paths`.
|
||||
|
||||
**Wiring:** both constructed where the autoscan handler is built in `router.go`
|
||||
(`deps.FolderRepo` already in scope; the arr client is trivial). The handler's
|
||||
service interface gains `SuggestRewrites`.
|
||||
|
||||
**Errors:** instance not found → 404; arr unreachable / bad key → 502 with the
|
||||
arr error surfaced; Silo folder-list failure → 500.
|
||||
|
||||
No new write path — the frontend merges `proposed` into the editor and the admin
|
||||
saves via the existing `PUT /sources/{id}`.
|
||||
|
||||
### 3. Frontend (preview UI)
|
||||
|
||||
In `AutoscanSourceEditor` (`web/src/pages/AdminRequests.tsx`), add a **"Sync from
|
||||
arr"** button beside Save. On click, a new hook
|
||||
(`useAutoscanRewriteSuggestions`) calls the endpoint and opens a preview panel:
|
||||
|
||||
- **Proposed** — each row: `From → To` + a **confidence badge** from `match_depth`
|
||||
(e.g. "3 segments" strong, "1 segment — weak"), with a checkbox (default checked).
|
||||
- **Unmatched** — info-only list of arr roots with no Silo match ("add manually").
|
||||
- **Ambiguous** — arr root + candidate Silo paths (info-only; admin adds a manual row).
|
||||
|
||||
**"Add selected to rewrites"** merges the checked proposals into the editor's
|
||||
existing rewrite rows, **deduped by `from`, never removing manual rows**. The
|
||||
admin then clicks the normal **Save**. New types in `web/src/api/types.ts`
|
||||
(`AutoscanRewriteSuggestions`, `ProposedRewrite`, `AmbiguousRoot`) and hook in
|
||||
`web/src/hooks/queries/useAutoscan.ts`. Reuse existing
|
||||
`Button`/`Badge`/`Dialog`/`Field` primitives.
|
||||
|
||||
### 4. Testing
|
||||
|
||||
- **`suggestRewrites`** — table-driven, fixtures spanning layouts: multi-segment
|
||||
match, single-segment unique, longest-suffix disambiguation (`Anime/Subs` vs
|
||||
`Anime2/Subs`), unmatched (no shared segment), already-covered via a broad
|
||||
prefix rule, an ambiguous tie, plus **unlike-this-deployment** cases
|
||||
(`/data/Movies` arr vs `/library/Films` Silo → unmatched; `/srv/tv/Show` vs
|
||||
`/tank/television/Show` → leaf match) and normalization (trailing slash,
|
||||
backslashes, duplicate slashes).
|
||||
- **`Service.SuggestRewrites`** — fakes for source store / `RootFolderClient` /
|
||||
`FolderLister` / secrets: asserts proposed/unmatched/ambiguous; arr-unreachable
|
||||
surfaces the error.
|
||||
- **Handler** — returns the JSON; 404 for missing instance; 502 when the arr is
|
||||
unreachable.
|
||||
- **Frontend** — the button hits the endpoint; the preview renders all three
|
||||
buckets with confidence badges; "Add selected" merges deduped rows; Save
|
||||
persists via the existing mutation.
|
||||
|
||||
## Risks & open considerations
|
||||
|
||||
- **Weak (1-segment) matches** can be coincidental across very different layouts;
|
||||
mitigated by surfacing `match_depth` so the admin rejects them before saving.
|
||||
- **Case-sensitivity** means case-mismatched mounts produce Unmatched rather than
|
||||
a wrong auto-match — a deliberate safe default; revisit only if requested.
|
||||
- **Arr reachability** is required for sync (unlike the rest of autoscan which
|
||||
polls on a schedule); a 502 with the arr error tells the admin to check the
|
||||
instance, and manual entry still works.
|
||||
|
||||
## References
|
||||
|
||||
- Reused: `internal/requests/arrclient.ListRootFolders`,
|
||||
`internal/catalog/FolderRepository.List`, `internal/autoscan` (`PathRewrite`,
|
||||
`normalizeSeparators`, `Repository`, `Service`, the existing
|
||||
`PUT /autoscan/sources/{id}` save).
|
||||
@@ -0,0 +1,161 @@
|
||||
# Autoscan source labels — design
|
||||
|
||||
**Date:** 2026-06-05
|
||||
**Status:** Approved, pending implementation plan
|
||||
**Scope:** Admin UI labeling of autoscan scan sources, generic across plugins, plus an operator-editable per-source label.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
Autoscan sources are labeled in the admin UI (Sources tab and Activity tab) by their
|
||||
`capability_id` plus `plugin #<installation_id>`. The `arr` scan-source plugin fans out
|
||||
one source per connection (Sonarr, Sonarr4k, Radarr, Radarr4k) under a single generic
|
||||
`arr` capability, so every row rendered an identical "arr (plugin #4)" label. A prior
|
||||
change made the Sources/Activity labels lead with the bound connection name, which fixes
|
||||
`arr`, but the fallback for any other plugin is still the cryptic `capability_id` +
|
||||
`plugin #N`.
|
||||
|
||||
Two gaps remain:
|
||||
|
||||
1. **Other plugins look cryptic.** A connectionless plugin (e.g. cephfs) or any future
|
||||
plugin still falls back to the raw `capability_id`, ignoring the human-friendly
|
||||
`display_name` the plugin already declares in its manifest.
|
||||
2. **No operator control.** An operator cannot name a source themselves. This is the only
|
||||
fully-general answer for plugin shapes the automatic chain cannot disambiguate (e.g. a
|
||||
single plugin installation creating several connectionless sources distinguished only
|
||||
by their `source_config`).
|
||||
|
||||
## Goals
|
||||
|
||||
- Generic, plugin-agnostic labels driven by metadata that already exists
|
||||
(manifest `display_name`, connection `name`), so future plugins look good with **no
|
||||
host code per plugin**.
|
||||
- An optional operator-editable label per source that overrides the automatic chain.
|
||||
- A single shared label-resolution helper, consumed by both the Sources and Activity
|
||||
panels, so the logic is not duplicated.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Plugin-declared label templates in the manifest/SDK (the "option C" from brainstorming).
|
||||
Deferred: it requires a cross-repo SDK/manifest contract change. The chain below leaves a
|
||||
clean seam for it.
|
||||
- Backend-side label resolution returned on API payloads. The resolution stays in the
|
||||
frontend helper for now; promote it to the backend only if the mobile clients need the
|
||||
same labels.
|
||||
|
||||
## Resolution chain
|
||||
|
||||
A source's display label resolves through four rungs, most-specific first:
|
||||
|
||||
```
|
||||
operator label -> connection name -> manifest display_name -> capability_id
|
||||
```
|
||||
|
||||
- **operator label** — the optional per-source text the admin sets (this design's new field).
|
||||
- **connection name** — the operator-named `autoscan_connections.name` bound to the source.
|
||||
- **manifest display_name** — from the plugin manifest, surfaced by the
|
||||
`scan-source-plugins` endpoint (`AutoscanAvailableSource.display_name`).
|
||||
- **capability_id** — last-resort raw identifier; never worse than today.
|
||||
|
||||
The chosen rung is the **title**. The remaining identity (plugin display name / capability +
|
||||
`plugin #<installation_id>`) renders as a **subtitle**, so no information is lost when a
|
||||
higher rung wins. Example: operator label `4K Movies` as title, `Radarr4k · plugin #4` as
|
||||
subtitle.
|
||||
|
||||
### Per-plugin behavior
|
||||
|
||||
| Source shape | Title | Subtitle |
|
||||
| --- | --- | --- |
|
||||
| arr, connection bound, no operator label | `Radarr4k` (connection) | `arr · plugin #4` |
|
||||
| arr, operator label set | `4K Movies` (operator) | `Radarr4k · plugin #4` |
|
||||
| cephfs, no connection | `CephFS Watcher` (manifest display_name) | `plugin #5` |
|
||||
| future plugin, nothing declared | `capability_id` | `plugin #N` |
|
||||
| Activity event whose source was deleted (`source_id` null) | manifest display_name or `capability_id` from the event's stored fields | `plugin #N` |
|
||||
|
||||
## Components
|
||||
|
||||
### Shared helper — `web/src/lib/autoscanLabels.ts` (new)
|
||||
|
||||
Pure, React-free, unit-tested.
|
||||
|
||||
- `composeSourceLabel({ operatorLabel, connectionName, displayName, capabilityId, installationId })`
|
||||
`=> { name: string; detail: string }` — implements the four-rung chain and the
|
||||
subtitle composition.
|
||||
- `pluginDisplayNameKey(installationId, capabilityId) => string` — stable map key.
|
||||
- `buildPluginDisplayNames(available: AutoscanAvailableSource[]) => Map<string, string>` —
|
||||
builds the `installation_id + capability_id -> display_name` lookup both panels use.
|
||||
|
||||
Future seam: `composeSourceLabel` may later accept an optional `configLabel` rung (for the
|
||||
deferred plugin-template option) without changing either panel's call shape.
|
||||
|
||||
### Migration — `migrations/174_autoscan_source_label.{up,down}.sql` (new)
|
||||
|
||||
```sql
|
||||
-- up
|
||||
ALTER TABLE public.autoscan_sources ADD COLUMN label text NOT NULL DEFAULT '';
|
||||
-- down
|
||||
ALTER TABLE public.autoscan_sources DROP COLUMN label;
|
||||
```
|
||||
|
||||
`label` empty string = unset; the resolution chain falls through to the automatic rungs.
|
||||
174 is a fresh, unused migration number (no collision with the reused-172 history).
|
||||
|
||||
### Backend (Go)
|
||||
|
||||
- `internal/autoscan/types.go` — add `Label string` to the `Source` struct and the source
|
||||
input type.
|
||||
- `internal/autoscan/repository.go` — add `label` to `sourceColumns`, `scanSource`, and the
|
||||
create/update statements.
|
||||
- `internal/api/handlers/autoscan.go` — accept `label` on source update; include it in the
|
||||
`autoscanStatusSource` and source responses. Trim whitespace and cap length at 120
|
||||
characters server-side.
|
||||
|
||||
### Frontend
|
||||
|
||||
- `web/src/api/types.ts` — add `label` to `AutoscanSource` and `AutoscanSourceInput`.
|
||||
- `web/src/pages/admin/autoscan/SourcesPanel.tsx` — replace the inline `sourceIdentity`
|
||||
block with a call to `composeSourceLabel`. Add an optional "Label" text input to the
|
||||
source row, saved on blur via the existing `fullBody` PATCH path (mirrors the interval
|
||||
field's `handleIntervalBlur` auto-save). Build the display-name map from the existing
|
||||
`useAvailableScanSources()` query.
|
||||
- `web/src/pages/admin/autoscan/ActivityPanel.tsx` — add `useAvailableScanSources()`, build
|
||||
the display-name map, map `source_id -> source` to obtain the operator label and
|
||||
connection, and route `pollSourceName` / `scanSourceName` through `composeSourceLabel`,
|
||||
returning `.name`. Preserve the existing `"Autoscan"` fallback for scans with no
|
||||
capability.
|
||||
|
||||
## Edit UX
|
||||
|
||||
Inline per-row, reusing the existing on-blur auto-save pattern (no modal, no new
|
||||
interaction model). The label input is blank by default with placeholder text indicating
|
||||
it is optional. Saving sends the full row state through the same source-update mutation
|
||||
used for connection/interval/rewrites.
|
||||
|
||||
## Error handling and edge cases
|
||||
|
||||
- **Deleted source on an Activity event** — `autoscan_events.source_id` is
|
||||
`ON DELETE SET NULL`; with a null `source_id` there is no operator label or connection,
|
||||
so the chain resolves from the event row's own stored `capability_id` / `installation_id`
|
||||
(plus manifest display_name if the plugin is still installed).
|
||||
- **Uninstalled plugin** — `display_name` does not resolve; the chain falls to
|
||||
`capability_id`.
|
||||
- **Bound connection** — `autoscan_sources.connection_id` is `ON DELETE RESTRICT`, so a
|
||||
bound connection always resolves to a name.
|
||||
- **Length / whitespace** — the operator label is trimmed and capped at 120 characters
|
||||
server-side; an all-whitespace label normalizes to empty (unset).
|
||||
|
||||
## Testing
|
||||
|
||||
- `web/src/lib/autoscanLabels.test.ts` — covers each rung of the chain: operator label
|
||||
wins, connection name wins, manifest display_name wins, capability fallback, and the
|
||||
subtitle composition.
|
||||
- Extend the existing autoscan repository/handler Go tests to cover the `label` column
|
||||
round-trip (create/update/read) and the server-side trim + length cap.
|
||||
|
||||
## Scope summary
|
||||
|
||||
One migration pair, three Go files, four frontend files (one new). No SDK changes, no
|
||||
multi-repo coordination. Operator-editable labels (this design) and the automatic chain
|
||||
share the same helper, so the deferred plugin-template option layers on later with no
|
||||
rework.
|
||||
@@ -71,7 +71,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.0
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
|
||||
@@ -2,8 +2,8 @@ entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.0 h1:DJkRROQfr/kfwnF5dUdkhmbCso1KDLZc7uK7YfJnaO0=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220 h1:tiwfULdh9yOW7HO6fYf8v+lXIsrCrGfp3GM8ymhrBME=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package adminjob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// CancelRegistry bridges HTTP cancellation requests to in-process job contexts.
|
||||
// Queued jobs are cancelled in the repository; running jobs need their active
|
||||
// context cancelled so executors stop doing work before the row is finalized.
|
||||
type CancelRegistry struct {
|
||||
mu sync.Mutex
|
||||
cancels map[string]cancelEntry
|
||||
}
|
||||
|
||||
type cancelEntry struct {
|
||||
cancel context.CancelFunc
|
||||
token *struct{}
|
||||
}
|
||||
|
||||
func NewCancelRegistry() *CancelRegistry {
|
||||
return &CancelRegistry{cancels: make(map[string]cancelEntry)}
|
||||
}
|
||||
|
||||
func (r *CancelRegistry) Register(jobID string, cancel context.CancelFunc) func() {
|
||||
if r == nil || jobID == "" || cancel == nil {
|
||||
return func() {}
|
||||
}
|
||||
token := &struct{}{}
|
||||
r.mu.Lock()
|
||||
r.cancels[jobID] = cancelEntry{cancel: cancel, token: token}
|
||||
r.mu.Unlock()
|
||||
|
||||
return func() {
|
||||
r.mu.Lock()
|
||||
if entry := r.cancels[jobID]; entry.token == token {
|
||||
delete(r.cancels, jobID)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *CancelRegistry) Cancel(jobID string) bool {
|
||||
if r == nil || jobID == "" {
|
||||
return false
|
||||
}
|
||||
r.mu.Lock()
|
||||
cancel := r.cancels[jobID].cancel
|
||||
r.mu.Unlock()
|
||||
if cancel == nil {
|
||||
return false
|
||||
}
|
||||
cancel()
|
||||
return true
|
||||
}
|
||||
@@ -542,17 +542,19 @@ func (e *ItemRefreshExecutor) Execute(ctx context.Context, req ItemRefreshReques
|
||||
e.publish(cache.EventMetadataUpdated, refreshContentID)
|
||||
if e.realtimeHub != nil {
|
||||
if scanResult != nil && (scanResult.New > 0 || scanResult.Updated > 0 || scanResult.Missing > 0 || matched > 0) {
|
||||
_ = e.realtimeHub.PublishLibraryChanged(ctx, notifications.LibraryChangeEvent{
|
||||
_ = e.realtimeHub.PublishCatalogLibraryChanged(ctx, notifications.LibraryChangeEvent{
|
||||
LibraryID: req.ScanFolderID,
|
||||
Reason: "item_refresh",
|
||||
New: scanResult.New,
|
||||
Updated: scanResult.Updated,
|
||||
Missing: scanResult.Missing,
|
||||
MatchedFiles: matched,
|
||||
})
|
||||
}
|
||||
_ = e.realtimeHub.PublishMetadataUpdated(ctx, notifications.MetadataUpdateEvent{
|
||||
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
|
||||
LibraryID: req.ScanFolderID,
|
||||
ContentID: refreshContentID,
|
||||
Change: "metadata_updated",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -251,8 +251,9 @@ func (e *LibraryRefreshExecutor) Execute(
|
||||
if result.RefreshedOK > 0 || result.PipelineOK > 0 {
|
||||
e.publish(cache.EventMetadataUpdated, strconv.Itoa(req.LibraryID))
|
||||
if e.realtimeHub != nil {
|
||||
_ = e.realtimeHub.PublishMetadataUpdated(ctx, notifications.MetadataUpdateEvent{
|
||||
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
|
||||
LibraryID: req.LibraryID,
|
||||
Change: "metadata_updated",
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -321,6 +322,7 @@ func (e *LibraryRefreshExecutor) refreshItemsWithIDs(
|
||||
result.RefreshedFailed++
|
||||
} else {
|
||||
result.RefreshedOK++
|
||||
e.publishCatalogItemChanged(ctx, result.LibraryID, itemResult.contentID)
|
||||
}
|
||||
advance("Refreshing items with external IDs")
|
||||
}
|
||||
@@ -386,6 +388,7 @@ func (e *LibraryRefreshExecutor) refreshUnmatchedItem(ctx context.Context, libra
|
||||
if err := e.refresher.RefreshItemForLibrary(ctx, req.RefreshContentID, req.ScanFolderID); err != nil {
|
||||
return fmt.Errorf("refresh metadata: %w", err)
|
||||
}
|
||||
e.publishCatalogItemChanged(ctx, libraryID, req.RefreshContentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -399,6 +402,17 @@ func (e *LibraryRefreshExecutor) publish(eventType, payload string) {
|
||||
})
|
||||
}
|
||||
|
||||
func (e *LibraryRefreshExecutor) publishCatalogItemChanged(ctx context.Context, libraryID int, contentID string) {
|
||||
if e == nil || e.realtimeHub == nil || contentID == "" {
|
||||
return
|
||||
}
|
||||
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
|
||||
LibraryID: libraryID,
|
||||
ContentID: contentID,
|
||||
Change: "metadata_updated",
|
||||
})
|
||||
}
|
||||
|
||||
func decodeLibraryRefreshRequest(data json.RawMessage) (LibraryRefreshRequest, error) {
|
||||
var req LibraryRefreshRequest
|
||||
if len(data) == 0 {
|
||||
|
||||
@@ -24,11 +24,13 @@ const (
|
||||
StatusRunning = "running"
|
||||
StatusCompleted = "completed"
|
||||
StatusFailed = "failed"
|
||||
StatusCancelled = "cancelled"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrJobNotFound = errors.New("admin job not found")
|
||||
ErrActiveJobConflict = errors.New("admin job already active for type")
|
||||
ErrJobNotCancellable = errors.New("admin job is not cancellable")
|
||||
)
|
||||
|
||||
type ActiveJobConflictError struct {
|
||||
@@ -471,6 +473,83 @@ func (r *Repository) Fail(ctx context.Context, id string, input FailJobInput) er
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Repository) Cancel(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error) {
|
||||
if message == "" {
|
||||
message = "Admin job cancelled"
|
||||
}
|
||||
job, err := scanAdminJob(r.pool.QueryRow(ctx, `
|
||||
UPDATE admin_jobs
|
||||
SET status = $2,
|
||||
message = $3,
|
||||
error_message = '',
|
||||
completed_at = NOW(),
|
||||
heartbeat_at = NOW(),
|
||||
expires_at = $4,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
AND status IN ($5, $6)
|
||||
RETURNING `+adminJobColumns,
|
||||
id,
|
||||
StatusCancelled,
|
||||
message,
|
||||
expiresAt,
|
||||
StatusQueued,
|
||||
StatusRunning,
|
||||
))
|
||||
if err == nil {
|
||||
return job, nil
|
||||
}
|
||||
if !errors.Is(err, ErrJobNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
existing, lookupErr := r.GetByID(ctx, id)
|
||||
if lookupErr != nil {
|
||||
return nil, lookupErr
|
||||
}
|
||||
if existing.Status != StatusQueued && existing.Status != StatusRunning {
|
||||
return nil, ErrJobNotCancellable
|
||||
}
|
||||
return nil, ErrJobNotFound
|
||||
}
|
||||
|
||||
func (r *Repository) CancelQueued(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error) {
|
||||
if message == "" {
|
||||
message = "Admin job cancelled"
|
||||
}
|
||||
job, err := scanAdminJob(r.pool.QueryRow(ctx, `
|
||||
UPDATE admin_jobs
|
||||
SET status = $2,
|
||||
message = $3,
|
||||
error_message = '',
|
||||
completed_at = NOW(),
|
||||
heartbeat_at = NOW(),
|
||||
expires_at = $4,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
AND status = $5
|
||||
RETURNING `+adminJobColumns,
|
||||
id,
|
||||
StatusCancelled,
|
||||
message,
|
||||
expiresAt,
|
||||
StatusQueued,
|
||||
))
|
||||
if err == nil {
|
||||
return job, nil
|
||||
}
|
||||
if !errors.Is(err, ErrJobNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
existing, lookupErr := r.GetByID(ctx, id)
|
||||
if lookupErr != nil {
|
||||
return nil, lookupErr
|
||||
}
|
||||
if existing.Status != StatusQueued {
|
||||
return nil, ErrJobNotCancellable
|
||||
}
|
||||
return nil, ErrJobNotFound
|
||||
}
|
||||
|
||||
func (r *Repository) RequeueStaleRunning(ctx context.Context, before time.Time) (int, error) {
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE admin_jobs
|
||||
|
||||
@@ -57,6 +57,7 @@ type Runner struct {
|
||||
heartbeatInterval time.Duration
|
||||
staleAfter time.Duration
|
||||
retention time.Duration
|
||||
cancelRegistry *CancelRegistry
|
||||
stop chan struct{}
|
||||
stopOnce sync.Once
|
||||
}
|
||||
@@ -99,10 +100,17 @@ func NewRunner(
|
||||
heartbeatInterval: 10 * time.Second,
|
||||
staleAfter: 2 * time.Minute,
|
||||
retention: 7 * 24 * time.Hour,
|
||||
cancelRegistry: NewCancelRegistry(),
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) SetCancelRegistry(registry *CancelRegistry) {
|
||||
if registry != nil {
|
||||
r.cancelRegistry = registry
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Start() {
|
||||
go func() {
|
||||
r.requeueStaleJobs()
|
||||
@@ -329,6 +337,8 @@ func (r *Runner) executeLibraryRefresh(job *models.AdminJob) {
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), libraryRefreshTimeout)
|
||||
defer cancel()
|
||||
unregisterCancel := r.cancelRegistry.Register(job.ID, cancel)
|
||||
defer unregisterCancel()
|
||||
|
||||
heartbeatStop := make(chan struct{})
|
||||
go r.heartbeatLoop(ctx, job.ID, heartbeatStop)
|
||||
@@ -357,6 +367,10 @@ func (r *Runner) executeLibraryRefresh(job *models.AdminJob) {
|
||||
r.publishJobByID(ctx, notifications.TypeJobProgress, job.ID)
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
r.cancelJob(job.ID, current, total, "Library metadata refresh cancelled")
|
||||
return
|
||||
}
|
||||
msg := err.Error()
|
||||
if ctx.Err() != nil {
|
||||
msg = fmt.Sprintf("timed out after %s: %s", libraryRefreshTimeout, msg)
|
||||
@@ -846,3 +860,21 @@ func (r *Runner) failJob(id string, current, total int, message, errorMessage st
|
||||
}
|
||||
r.publishJobByID(ctx, notifications.TypeJobFailed, id)
|
||||
}
|
||||
|
||||
func (r *Runner) cancelJob(id string, current, total int, message string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := r.repo.UpdateProgress(ctx, id, current, total, message); err != nil {
|
||||
slog.Warn("admin jobs: failed to update cancellation progress", "job_id", id, "error", err)
|
||||
}
|
||||
job, err := r.repo.Cancel(ctx, id, message, time.Now().UTC().Add(r.retention))
|
||||
if err != nil {
|
||||
slog.Warn("admin jobs: failed to mark job cancelled", "job_id", id, "error", err)
|
||||
return
|
||||
}
|
||||
if r.realtimeHub != nil {
|
||||
if err := r.realtimeHub.PublishJob(ctx, notifications.TypeJobCancelled, job); err != nil {
|
||||
slog.Warn("admin jobs: failed to publish job cancellation", "job_id", id, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/autoscan"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
mediarequests "github.com/Silo-Server/silo-server/internal/requests"
|
||||
"github.com/Silo-Server/silo-server/internal/scantrigger"
|
||||
)
|
||||
|
||||
// autoscanQueuer is the scan-enqueue surface BuildAutoscanService needs; it is
|
||||
// satisfied by *scanqueue.Service (autoscan.Queuer's concrete production impl).
|
||||
type autoscanQueuer = autoscan.Queuer
|
||||
|
||||
// RequestIntegrationLookup adapts the Requests repository to the autoscan
|
||||
// connection resolver's RequestIntegrationLookup: it resolves a soft-linked
|
||||
// Requests integration to its base URL and (write-only) api-key ref.
|
||||
type RequestIntegrationLookup struct {
|
||||
Repo *mediarequests.Repository
|
||||
}
|
||||
|
||||
func (l RequestIntegrationLookup) Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error) {
|
||||
integration, err := l.Repo.GetIntegration(ctx, integrationID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
// A reused Requests connection must honor the integration's live state. The
|
||||
// v1 poll gated on `WHERE ri.enabled = true`; here we surface a disabled or
|
||||
// unconfigured (blank base_url) integration as an error so the engine turns
|
||||
// it into a logged skip / RecordError rather than polling an unusable target.
|
||||
if err := checkRequestIntegrationUsable(integrationID, integration.Enabled, integration.BaseURL); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return integration.BaseURL, integration.APIKeyRef, nil
|
||||
}
|
||||
|
||||
// checkRequestIntegrationUsable returns a non-nil error when a linked Requests
|
||||
// integration cannot be polled: it is disabled, or it has no base_url. Extracted
|
||||
// as a pure function so the gating is unit-testable without a DB-backed repo.
|
||||
func checkRequestIntegrationUsable(integrationID string, enabled bool, baseURL string) error {
|
||||
if !enabled {
|
||||
return fmt.Errorf("linked requests integration %q is disabled", integrationID)
|
||||
}
|
||||
if strings.TrimSpace(baseURL) == "" {
|
||||
return fmt.Errorf("linked requests integration %q has no base_url configured", integrationID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PluginScanSourceAdapter adapts plugins.Service to autoscan.ScanSourceResolver.
|
||||
// plugins.Service.ScanSourceClient returns the concrete
|
||||
// *pluginhost.ScanSourceClient; that concrete type satisfies
|
||||
// autoscan.PollChangesClient (it has the matching PollChanges method), so the
|
||||
// adapter declares the exported interface as its return type and returns the
|
||||
// concrete value (Go has no return-type covariance, so the method signature must
|
||||
// name the interface exactly to satisfy ScanSourceResolver).
|
||||
type PluginScanSourceAdapter struct {
|
||||
Svc *plugins.Service
|
||||
}
|
||||
|
||||
func (a PluginScanSourceAdapter) ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (autoscan.PollChangesClient, error) {
|
||||
return a.Svc.ScanSourceClient(ctx, installationID, capabilityID)
|
||||
}
|
||||
|
||||
// scanSourceCapabilityType is the plugin capability type autoscan discovery
|
||||
// enumerates.
|
||||
const scanSourceCapabilityType = "scan_source.v1"
|
||||
|
||||
// PluginScanSourceLister adapts the plugin installation store to
|
||||
// autoscan.ScanSourceLister: it enumerates every installed scan_source.v1
|
||||
// capability across ALL installed plugins, regardless of enabled state.
|
||||
//
|
||||
// Using List (not ListEnabled) is deliberate for orphan detection: a temporarily
|
||||
// DISABLED-but-installed plugin must NOT be treated as orphaned. If we dropped
|
||||
// disabled plugins here, PollOnce would prune their sources and skip them with no
|
||||
// last_error — they'd vanish silently. Keeping them present means PollOnce still
|
||||
// attempts them; the plugin client fails to load (not running) and the source
|
||||
// gets a visible RecordError instead. Only a fully UNINSTALLED plugin (gone from
|
||||
// List entirely) is a true orphan. The Add-source picker shares this all-installed
|
||||
// set, which is fine — operators can bind sources against an installed capability.
|
||||
type PluginScanSourceLister struct {
|
||||
Store *plugins.InstallationStore
|
||||
}
|
||||
|
||||
func (l PluginScanSourceLister) ListScanSources(ctx context.Context) ([]autoscan.DiscoveredSource, error) {
|
||||
installations, err := l.Store.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []autoscan.DiscoveredSource
|
||||
for _, inst := range installations {
|
||||
caps, err := l.Store.ListCapabilities(ctx, inst.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range caps {
|
||||
if c == nil || c.Type != scanSourceCapabilityType {
|
||||
continue
|
||||
}
|
||||
out = append(out, autoscan.DiscoveredSource{
|
||||
InstallationID: c.InstallationID,
|
||||
CapabilityID: c.ID,
|
||||
PluginID: inst.PluginID,
|
||||
DisplayName: scanSourceDisplayName(inst.PluginID, c),
|
||||
})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scanSourceDisplayName derives a human-friendly label for a scan_source
|
||||
// capability: the capability manifest's display_name when present, else the
|
||||
// plugin id (with the capability id appended when it adds information).
|
||||
func scanSourceDisplayName(pluginID string, c *plugins.Capability) string {
|
||||
if c != nil && c.Metadata != nil {
|
||||
if name, ok := c.Metadata["display_name"].(string); ok && strings.TrimSpace(name) != "" {
|
||||
return strings.TrimSpace(name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case pluginID != "" && c != nil && c.ID != "":
|
||||
return pluginID + " / " + c.ID
|
||||
case pluginID != "":
|
||||
return pluginID
|
||||
case c != nil:
|
||||
return c.ID
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// AutoscanSecretResolver resolves an encrypted api-key reference to plaintext.
|
||||
// It is satisfied by the server settings repo (Get(ctx, key) (string, error)).
|
||||
type AutoscanSecretResolver interface {
|
||||
Get(ctx context.Context, ref string) (string, error)
|
||||
}
|
||||
|
||||
// BuildAutoscanService wires the v2 autoscan engine from its concrete
|
||||
// dependencies. Both the HTTP router (manual trigger) and the background poll
|
||||
// task share this constructor so the adapter wiring lives in exactly one place.
|
||||
func BuildAutoscanService(
|
||||
repo *autoscan.Repository,
|
||||
pluginService *plugins.Service,
|
||||
installationStore *plugins.InstallationStore,
|
||||
requestsRepo *mediarequests.Repository,
|
||||
secrets AutoscanSecretResolver,
|
||||
folderRepo *catalog.FolderRepository,
|
||||
queue autoscanQueuer,
|
||||
redisClient *redis.Client,
|
||||
) *autoscan.Service {
|
||||
provider := autoscan.NewPluginProvider(PluginScanSourceAdapter{pluginService})
|
||||
connRes := autoscan.NewConnectionResolver(RequestIntegrationLookup{requestsRepo}, secrets)
|
||||
svc := autoscan.NewService(
|
||||
repo,
|
||||
provider,
|
||||
connRes,
|
||||
scantrigger.NewResolver(folderRepo),
|
||||
queue,
|
||||
autoscan.NewRedisSuppressor(redisClient),
|
||||
PluginScanSourceLister{installationStore},
|
||||
)
|
||||
// Wire the connection-test + rewrite-suggester deps: a (long-timeout)
|
||||
// arr root-folder/status client and a Silo media-folder lister.
|
||||
svc.SetSuggesterDeps(
|
||||
autoscan.NewArrRootFolderClient(nil),
|
||||
autoscan.NewCatalogFolderLister(folderRepo),
|
||||
)
|
||||
return svc
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package api
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestCheckRequestIntegrationUsable verifies the Fix 4 gate: a reused Requests
|
||||
// connection is rejected when the integration is disabled or has no base_url, so
|
||||
// the autoscan engine skips it instead of polling an unusable target.
|
||||
func TestCheckRequestIntegrationUsable(t *testing.T) {
|
||||
t.Run("enabled with base_url is usable", func(t *testing.T) {
|
||||
if err := checkRequestIntegrationUsable("req-1", true, "http://radarr:7878"); err != nil {
|
||||
t.Fatalf("expected usable, got %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("disabled is rejected", func(t *testing.T) {
|
||||
if err := checkRequestIntegrationUsable("req-1", false, "http://radarr:7878"); err == nil {
|
||||
t.Fatal("expected error for disabled integration")
|
||||
}
|
||||
})
|
||||
t.Run("blank base_url is rejected", func(t *testing.T) {
|
||||
if err := checkRequestIntegrationUsable("req-1", true, " "); err == nil {
|
||||
t.Fatal("expected error for blank base_url")
|
||||
}
|
||||
})
|
||||
}
|
||||
+226
-52
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -373,6 +374,9 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
req.Username = auth.NormalizeUsername(req.Username)
|
||||
req.Email = auth.NormalizeEmail(req.Email)
|
||||
|
||||
if req.Username == "" || req.Email == "" || req.Password == "" || req.Role == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Username, email, password, and role are required")
|
||||
return
|
||||
@@ -460,7 +464,7 @@ func (h *AdminHandler) HandleUpdateUser(w http.ResponseWriter, r *http.Request)
|
||||
permissions = &normalized
|
||||
}
|
||||
|
||||
err = h.userRepo.Update(r.Context(), id, models.UpdateUserInput{
|
||||
updateInput := models.UpdateUserInput{
|
||||
Username: req.Username,
|
||||
Email: req.Email,
|
||||
Password: req.Password,
|
||||
@@ -474,12 +478,31 @@ func (h *AdminHandler) HandleUpdateUser(w http.ResponseWriter, r *http.Request)
|
||||
MaxProfiles: req.MaxProfiles,
|
||||
DownloadAllowed: req.DownloadAllowed,
|
||||
DownloadTranscodeAllowed: req.DownloadTranscodeAllowed,
|
||||
})
|
||||
}
|
||||
|
||||
var currentUser *models.User
|
||||
if updateMayRequireSessionRevocation(updateInput) {
|
||||
currentUser, err = h.userRepo.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
if auth.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "User not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch user")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = h.userRepo.Update(r.Context(), id, updateInput)
|
||||
if err != nil {
|
||||
if auth.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "User not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update user")
|
||||
return
|
||||
}
|
||||
if updateRequiresSessionRevocation(req) {
|
||||
if updateRequiresSessionRevocation(currentUser, updateInput) {
|
||||
if err := h.revokeUserSessions(r.Context(), id); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to revoke updated user sessions")
|
||||
return
|
||||
@@ -747,13 +770,35 @@ func (h *AdminHandler) HandleListUserProfiles(w http.ResponseWriter, r *http.Req
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func updateRequiresSessionRevocation(req updateUserRequest) bool {
|
||||
return req.Password != nil ||
|
||||
req.Role != nil ||
|
||||
req.Enabled != nil ||
|
||||
req.LibraryIDs.Set ||
|
||||
req.Permissions.Set ||
|
||||
req.MaxPlaybackQuality != nil
|
||||
func updateMayRequireSessionRevocation(input models.UpdateUserInput) bool {
|
||||
return input.Password != nil ||
|
||||
input.Role != nil ||
|
||||
input.Enabled != nil ||
|
||||
input.Permissions != nil ||
|
||||
input.MaxPlaybackQuality != nil
|
||||
}
|
||||
|
||||
func updateRequiresSessionRevocation(current *models.User, input models.UpdateUserInput) bool {
|
||||
if input.Password != nil {
|
||||
return true
|
||||
}
|
||||
if current == nil {
|
||||
return updateMayRequireSessionRevocation(input)
|
||||
}
|
||||
if input.Role != nil && *input.Role != current.Role {
|
||||
return true
|
||||
}
|
||||
if input.Enabled != nil && *input.Enabled != current.Enabled {
|
||||
return true
|
||||
}
|
||||
if input.Permissions != nil && !slices.Equal(*input.Permissions, current.Permissions) {
|
||||
return true
|
||||
}
|
||||
if input.MaxPlaybackQuality != nil &&
|
||||
access.NormalizePlaybackQuality(*input.MaxPlaybackQuality) != access.NormalizePlaybackQuality(current.MaxPlaybackQuality) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *AdminHandler) revokeUserSessions(ctx context.Context, userID int) error {
|
||||
@@ -830,6 +875,9 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
var resp AdminStats
|
||||
|
||||
if h.StatsSource != nil {
|
||||
if isTruthyQuery(r.URL.Query().Get("refresh")) {
|
||||
h.StatsSource.Invalidate()
|
||||
}
|
||||
stats, err := h.StatsSource.Get(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get stats")
|
||||
@@ -856,6 +904,15 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func isTruthyQuery(value string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AdminHandler) invalidateStats(ctx context.Context, channel, eventType, payload string) {
|
||||
if h.StatsSource != nil {
|
||||
h.StatsSource.Invalidate()
|
||||
@@ -966,6 +1023,7 @@ type UpdateItemMetadataRequest struct {
|
||||
FirstAirDate *string `json:"first_air_date"`
|
||||
LastAirDate *string `json:"last_air_date"`
|
||||
AirTime *string `json:"air_time"`
|
||||
AirTimezone *string `json:"air_timezone"`
|
||||
AirDate *string `json:"air_date"`
|
||||
Status *string `json:"status"`
|
||||
RatingIMDB *float64 `json:"rating_imdb"`
|
||||
@@ -993,6 +1051,14 @@ func (h *AdminHandler) HandleUpdateItemMetadata(w http.ResponseWriter, r *http.R
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.AirTimezone != nil {
|
||||
trimmed := strings.TrimSpace(*req.AirTimezone)
|
||||
req.AirTimezone = &trimmed
|
||||
if !catalog.ValidateAirTimezone(trimmed) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "air_timezone must be a valid IANA timezone")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
upd := catalog.MetadataUpdate{
|
||||
Title: req.Title, SortTitle: req.SortTitle, OriginalTitle: req.OriginalTitle,
|
||||
@@ -1000,7 +1066,7 @@ func (h *AdminHandler) HandleUpdateItemMetadata(w http.ResponseWriter, r *http.R
|
||||
Year: req.Year, Runtime: req.Runtime,
|
||||
Genres: req.Genres, Studios: req.Studios, Networks: req.Networks, Countries: req.Countries,
|
||||
ReleaseDate: req.ReleaseDate, FirstAirDate: req.FirstAirDate, LastAirDate: req.LastAirDate,
|
||||
AirTime: req.AirTime,
|
||||
AirTime: req.AirTime, AirTimezone: req.AirTimezone,
|
||||
AirDate: req.AirDate, Status: req.Status,
|
||||
RatingIMDB: req.RatingIMDB, RatingTMDB: req.RatingTMDB,
|
||||
RatingRTCritic: req.RatingRTCritic, RatingRTAudience: req.RatingRTAudience,
|
||||
@@ -1070,6 +1136,7 @@ var sensitiveSettingKeys = map[string]bool{
|
||||
"mdblist.api_key": true,
|
||||
"requests.radarr.api_key": true,
|
||||
"requests.sonarr.api_key": true,
|
||||
"subtitle_ai.api_key": true,
|
||||
"watchsync.trakt.client_id": true,
|
||||
"watchsync.trakt.client_secret": true,
|
||||
"watchsync.simkl.client_id": true,
|
||||
@@ -1202,6 +1269,9 @@ type adminDeviceDetailResponse struct {
|
||||
DeviceID string `json:"device_id"`
|
||||
DeviceName string `json:"device_name"`
|
||||
DevicePlatform string `json:"device_platform"`
|
||||
OverrideCount int `json:"override_count"`
|
||||
ProfileCount int `json:"profile_count"`
|
||||
Profiles []adminDeviceProfileSummary `json:"profiles"`
|
||||
LastUpdated string `json:"last_updated"`
|
||||
Settings []adminDeviceSettingResponse `json:"settings"`
|
||||
}
|
||||
@@ -1444,6 +1514,12 @@ func (h *AdminHandler) HandleUpdateUserDeviceSetting(w http.ResponseWriter, r *h
|
||||
if existing != nil {
|
||||
entry.DeviceName = existing.DeviceName
|
||||
entry.DevicePlatform = existing.DevicePlatform
|
||||
} else if registered, err := registeredDeviceForProfile(r.Context(), store, profileID, deviceID); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
|
||||
return
|
||||
} else if registered != nil {
|
||||
entry.DeviceName = registered.DeviceName
|
||||
entry.DevicePlatform = registered.DevicePlatform
|
||||
}
|
||||
if err := store.SetDeviceSetting(r.Context(), entry); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update device setting")
|
||||
@@ -1566,6 +1642,10 @@ func (h *AdminHandler) HandleListDevices(w http.ResponseWriter, r *http.Request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list device settings: %w", err)
|
||||
}
|
||||
devices, err := listRegisteredDevices(gctx, store)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list devices: %w", err)
|
||||
}
|
||||
profileNames, err := listProfileNamesByID(gctx, store)
|
||||
if err != nil {
|
||||
slog.Warn("admin list devices profile lookup failed",
|
||||
@@ -1574,7 +1654,14 @@ func (h *AdminHandler) HandleListDevices(w http.ResponseWriter, r *http.Request)
|
||||
)
|
||||
profileNames = map[string]string{}
|
||||
}
|
||||
perUser[i] = buildAdminDeviceSummaries(user.ID, user.Username, user.Email, entries, profileNames)
|
||||
perUser[i] = buildAdminDeviceSummaries(
|
||||
user.ID,
|
||||
user.Username,
|
||||
user.Email,
|
||||
entries,
|
||||
devices,
|
||||
profileNames,
|
||||
)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -1638,6 +1725,11 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
|
||||
return
|
||||
}
|
||||
registeredDevices, err := listRegisteredDevices(r.Context(), store)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
|
||||
return
|
||||
}
|
||||
profileNames, err := listProfileNamesByID(r.Context(), store)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list profiles")
|
||||
@@ -1650,12 +1742,26 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
|
||||
deviceEntries = append(deviceEntries, entry)
|
||||
}
|
||||
}
|
||||
if len(deviceEntries) == 0 {
|
||||
deviceRegistrations := make([]userstore.DeviceEntry, 0)
|
||||
for _, entry := range registeredDevices {
|
||||
if entry.DeviceID == deviceID {
|
||||
deviceRegistrations = append(deviceRegistrations, entry)
|
||||
}
|
||||
}
|
||||
summaries := buildAdminDeviceSummaries(
|
||||
user.ID,
|
||||
user.Username,
|
||||
user.Email,
|
||||
deviceEntries,
|
||||
deviceRegistrations,
|
||||
profileNames,
|
||||
)
|
||||
if len(summaries) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Device not found")
|
||||
return
|
||||
}
|
||||
|
||||
summary := buildAdminDeviceSummaries(user.ID, user.Username, user.Email, deviceEntries, profileNames)[0]
|
||||
summary := summaries[0]
|
||||
writeJSON(w, http.StatusOK, adminDeviceDetailResponse{
|
||||
UserID: user.ID,
|
||||
Username: user.Username,
|
||||
@@ -1663,11 +1769,41 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
|
||||
DeviceID: summary.DeviceID,
|
||||
DeviceName: summary.DeviceName,
|
||||
DevicePlatform: summary.DevicePlatform,
|
||||
OverrideCount: summary.OverrideCount,
|
||||
ProfileCount: summary.ProfileCount,
|
||||
Profiles: summary.Profiles,
|
||||
LastUpdated: summary.LastUpdated,
|
||||
Settings: buildAdminDeviceSettingsResponse(user.ID, profileNames, deviceEntries).Settings,
|
||||
})
|
||||
}
|
||||
|
||||
func listRegisteredDevices(ctx context.Context, store userstore.UserStore) ([]userstore.DeviceEntry, error) {
|
||||
registry, ok := store.(userstore.DeviceRegistry)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return registry.ListDevices(ctx)
|
||||
}
|
||||
|
||||
func registeredDeviceForProfile(
|
||||
ctx context.Context,
|
||||
store userstore.UserStore,
|
||||
profileID string,
|
||||
deviceID string,
|
||||
) (*userstore.DeviceEntry, error) {
|
||||
devices, err := listRegisteredDevices(ctx, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, device := range devices {
|
||||
if device.ProfileID == profileID && device.DeviceID == deviceID {
|
||||
matched := device
|
||||
return &matched, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func buildAdminDeviceSettingsResponse(userID int, profileNames map[string]string, entries []userstore.DeviceSettingEntry) adminDeviceSettingsListResponse {
|
||||
resp := adminDeviceSettingsListResponse{
|
||||
Settings: make([]adminDeviceSettingResponse, 0, len(entries)),
|
||||
@@ -1693,6 +1829,7 @@ func buildAdminDeviceSummaries(
|
||||
username string,
|
||||
email string,
|
||||
entries []userstore.DeviceSettingEntry,
|
||||
registeredDevices []userstore.DeviceEntry,
|
||||
profileNames map[string]string,
|
||||
) []adminDeviceSummaryResponse {
|
||||
type profileAccumulator struct {
|
||||
@@ -1706,10 +1843,10 @@ func buildAdminDeviceSummaries(
|
||||
}
|
||||
|
||||
byDevice := make(map[string]*summary)
|
||||
for _, entry := range entries {
|
||||
deviceID := strings.TrimSpace(entry.DeviceID)
|
||||
|
||||
ensureDevice := func(deviceID, deviceName, devicePlatform, lastUpdated string) *summary {
|
||||
if deviceID == "" {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
current, ok := byDevice[deviceID]
|
||||
if !ok {
|
||||
@@ -1719,49 +1856,86 @@ func buildAdminDeviceSummaries(
|
||||
Username: username,
|
||||
Email: email,
|
||||
DeviceID: deviceID,
|
||||
DeviceName: entry.DeviceName,
|
||||
DevicePlatform: entry.DevicePlatform,
|
||||
LastUpdated: entry.UpdatedAt,
|
||||
DeviceName: deviceName,
|
||||
DevicePlatform: devicePlatform,
|
||||
LastUpdated: lastUpdated,
|
||||
},
|
||||
keys: make(map[string]struct{}),
|
||||
profiles: make(map[string]*profileAccumulator),
|
||||
}
|
||||
byDevice[deviceID] = current
|
||||
}
|
||||
current.keys[entry.ProfileID+":"+entry.Key] = struct{}{}
|
||||
if current.device.DeviceName == "" && deviceName != "" {
|
||||
current.device.DeviceName = deviceName
|
||||
}
|
||||
if current.device.DevicePlatform == "" && devicePlatform != "" {
|
||||
current.device.DevicePlatform = devicePlatform
|
||||
}
|
||||
if lastUpdated > current.device.LastUpdated {
|
||||
current.device.LastUpdated = lastUpdated
|
||||
if deviceName != "" {
|
||||
current.device.DeviceName = deviceName
|
||||
}
|
||||
if devicePlatform != "" {
|
||||
current.device.DevicePlatform = devicePlatform
|
||||
}
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
ensureProfile := func(current *summary, profileID, lastUpdated string) *profileAccumulator {
|
||||
if current == nil || profileID == "" {
|
||||
return nil
|
||||
}
|
||||
profile, exists := current.profiles[profileID]
|
||||
if !exists {
|
||||
profile = &profileAccumulator{
|
||||
summary: adminDeviceProfileSummary{
|
||||
ProfileID: profileID,
|
||||
ProfileName: profileNames[profileID],
|
||||
LastUpdated: lastUpdated,
|
||||
},
|
||||
keys: make(map[string]struct{}),
|
||||
}
|
||||
current.profiles[profileID] = profile
|
||||
return profile
|
||||
}
|
||||
if lastUpdated > profile.summary.LastUpdated {
|
||||
profile.summary.LastUpdated = lastUpdated
|
||||
}
|
||||
return profile
|
||||
}
|
||||
|
||||
for _, device := range registeredDevices {
|
||||
deviceID := strings.TrimSpace(device.DeviceID)
|
||||
profileID := strings.TrimSpace(device.ProfileID)
|
||||
current := ensureDevice(
|
||||
deviceID,
|
||||
device.DeviceName,
|
||||
device.DevicePlatform,
|
||||
device.LastSeenAt,
|
||||
)
|
||||
ensureProfile(current, profileID, device.LastSeenAt)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
deviceID := strings.TrimSpace(entry.DeviceID)
|
||||
profileID := strings.TrimSpace(entry.ProfileID)
|
||||
if profileID != "" {
|
||||
profile, exists := current.profiles[profileID]
|
||||
if !exists {
|
||||
profile = &profileAccumulator{
|
||||
summary: adminDeviceProfileSummary{
|
||||
ProfileID: profileID,
|
||||
ProfileName: profileNames[profileID],
|
||||
LastUpdated: entry.UpdatedAt,
|
||||
},
|
||||
keys: make(map[string]struct{}),
|
||||
}
|
||||
current.profiles[profileID] = profile
|
||||
}
|
||||
current := ensureDevice(
|
||||
deviceID,
|
||||
entry.DeviceName,
|
||||
entry.DevicePlatform,
|
||||
entry.UpdatedAt,
|
||||
)
|
||||
if current == nil {
|
||||
continue
|
||||
}
|
||||
if profileID != "" && entry.Key != "" {
|
||||
current.keys[profileID+":"+entry.Key] = struct{}{}
|
||||
}
|
||||
profile := ensureProfile(current, profileID, entry.UpdatedAt)
|
||||
if profile != nil && entry.Key != "" {
|
||||
profile.keys[entry.Key] = struct{}{}
|
||||
if entry.UpdatedAt > profile.summary.LastUpdated {
|
||||
profile.summary.LastUpdated = entry.UpdatedAt
|
||||
}
|
||||
}
|
||||
if current.device.DeviceName == "" && entry.DeviceName != "" {
|
||||
current.device.DeviceName = entry.DeviceName
|
||||
}
|
||||
if current.device.DevicePlatform == "" && entry.DevicePlatform != "" {
|
||||
current.device.DevicePlatform = entry.DevicePlatform
|
||||
}
|
||||
if entry.UpdatedAt > current.device.LastUpdated {
|
||||
current.device.LastUpdated = entry.UpdatedAt
|
||||
if entry.DeviceName != "" {
|
||||
current.device.DeviceName = entry.DeviceName
|
||||
}
|
||||
if entry.DevicePlatform != "" {
|
||||
current.device.DevicePlatform = entry.DevicePlatform
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
)
|
||||
|
||||
const adminJobDownloadExpiry = 15 * time.Minute
|
||||
@@ -23,12 +24,22 @@ type AdminJobArtifactStore interface {
|
||||
PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error)
|
||||
}
|
||||
|
||||
type AdminJobsHandler struct {
|
||||
repo *adminjob.Repository
|
||||
store AdminJobArtifactStore
|
||||
type adminJobRepository interface {
|
||||
List(ctx context.Context, opts adminjob.ListJobsOptions) ([]*models.AdminJob, error)
|
||||
GetByID(ctx context.Context, id string) (*models.AdminJob, error)
|
||||
Cancel(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error)
|
||||
CancelQueued(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error)
|
||||
UpdateProgress(ctx context.Context, id string, current, total int, message string) error
|
||||
}
|
||||
|
||||
func NewAdminJobsHandler(repo *adminjob.Repository, store AdminJobArtifactStore) *AdminJobsHandler {
|
||||
type AdminJobsHandler struct {
|
||||
repo adminJobRepository
|
||||
store AdminJobArtifactStore
|
||||
CancelRegistry *adminjob.CancelRegistry
|
||||
RealtimeHub *notifications.Hub
|
||||
}
|
||||
|
||||
func NewAdminJobsHandler(repo adminJobRepository, store AdminJobArtifactStore) *AdminJobsHandler {
|
||||
return &AdminJobsHandler{repo: repo, store: store}
|
||||
}
|
||||
|
||||
@@ -120,6 +131,105 @@ func (h *AdminJobsHandler) HandleGet(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, adminJobToResponseForClaims(r, job, h.store, claims))
|
||||
}
|
||||
|
||||
func (h *AdminJobsHandler) HandleCancel(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.repo == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "not_configured", "Admin jobs are not configured")
|
||||
return
|
||||
}
|
||||
|
||||
id := chi.URLParam(r, "id")
|
||||
if id == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Job ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
job, err := h.repo.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, adminjob.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
|
||||
return
|
||||
}
|
||||
|
||||
if job.JobType != adminjob.JobTypeLibraryRefresh {
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Only library metadata refresh jobs can be cancelled")
|
||||
return
|
||||
}
|
||||
|
||||
switch job.Status {
|
||||
case adminjob.StatusQueued:
|
||||
cancelled, cancelErr := h.repo.CancelQueued(
|
||||
r.Context(),
|
||||
id,
|
||||
"Library metadata refresh cancelled",
|
||||
time.Now().UTC().Add(7*24*time.Hour),
|
||||
)
|
||||
if cancelErr != nil {
|
||||
if errors.Is(cancelErr, adminjob.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(cancelErr, adminjob.ErrJobNotCancellable) {
|
||||
advanced, loadErr := h.repo.GetByID(r.Context(), id)
|
||||
if loadErr != nil {
|
||||
if errors.Is(loadErr, adminjob.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
|
||||
return
|
||||
}
|
||||
if advanced.Status == adminjob.StatusRunning {
|
||||
h.requestRunningCancellation(w, r, advanced)
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Job is no longer cancellable")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel job")
|
||||
return
|
||||
}
|
||||
if h.RealtimeHub != nil {
|
||||
publishEventJob(
|
||||
r.Context(),
|
||||
h.RealtimeHub.EventsHub(),
|
||||
string(notifications.TypeJobCancelled),
|
||||
cancelled,
|
||||
)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, adminJobToResponse(r, cancelled, h.store))
|
||||
case adminjob.StatusRunning:
|
||||
h.requestRunningCancellation(w, r, job)
|
||||
default:
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Job is no longer cancellable")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AdminJobsHandler) requestRunningCancellation(w http.ResponseWriter, r *http.Request, job *models.AdminJob) {
|
||||
if h.CancelRegistry == nil || !h.CancelRegistry.Cancel(job.ID) {
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Running job is not cancellable on this node")
|
||||
return
|
||||
}
|
||||
if err := h.repo.UpdateProgress(
|
||||
r.Context(),
|
||||
job.ID,
|
||||
job.ProgressCurrent,
|
||||
job.ProgressTotal,
|
||||
"Cancellation requested",
|
||||
); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to request cancellation")
|
||||
return
|
||||
}
|
||||
updated, err := h.repo.GetByID(r.Context(), job.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, adminJobToResponse(r, updated, h.store))
|
||||
}
|
||||
|
||||
func adminJobToResponse(r *http.Request, job *models.AdminJob, store AdminJobArtifactStore) adminJobResponse {
|
||||
resp := adminJobResponse{
|
||||
ID: job.ID,
|
||||
|
||||
@@ -2,12 +2,21 @@ package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/adminjob"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
)
|
||||
|
||||
func TestCanReadAdminJob_AdminCanReadAnyJob(t *testing.T) {
|
||||
@@ -79,3 +88,195 @@ func TestAdminJobToResponseForClaims_NonAdminSanitizesItemRefreshPayloads(t *tes
|
||||
t.Fatalf("ErrorMessage = %q, want stripped for non-admin", resp.ErrorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminJobsHandleCancel_RunningRegistryMissDoesNotUpdateProgress(t *testing.T) {
|
||||
repo := &fakeAdminJobRepository{
|
||||
job: &models.AdminJob{
|
||||
ID: "job-1",
|
||||
JobType: adminjob.JobTypeLibraryRefresh,
|
||||
Status: adminjob.StatusRunning,
|
||||
CreatedByUserID: 1,
|
||||
Message: "Running",
|
||||
RequestedAt: time.Now().UTC(),
|
||||
},
|
||||
}
|
||||
handler := NewAdminJobsHandler(repo, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
|
||||
|
||||
if rec.Code != http.StatusConflict {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusConflict)
|
||||
}
|
||||
if repo.updateProgressCalls != 0 {
|
||||
t.Fatalf("UpdateProgress called %d times, want 0", repo.updateProgressCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminJobsHandleCancel_QueuedPublishesCancelledEvent(t *testing.T) {
|
||||
repo := &fakeAdminJobRepository{
|
||||
job: &models.AdminJob{
|
||||
ID: "job-1",
|
||||
JobType: adminjob.JobTypeLibraryRefresh,
|
||||
Status: adminjob.StatusQueued,
|
||||
CreatedByUserID: 1,
|
||||
Message: "Queued",
|
||||
RequestedAt: time.Now().UTC(),
|
||||
},
|
||||
}
|
||||
hub := notifications.NewHub("test", &cache.NoopEventBus{})
|
||||
events, unsubscribe := hub.Subscribe()
|
||||
defer unsubscribe()
|
||||
|
||||
handler := NewAdminJobsHandler(repo, nil)
|
||||
handler.RealtimeHub = hub
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case event := <-events:
|
||||
if event.Type != notifications.TypeJobCancelled {
|
||||
t.Fatalf("event.Type = %q, want %q", event.Type, notifications.TypeJobCancelled)
|
||||
}
|
||||
if event.Job == nil || event.Job.ID != "job-1" {
|
||||
t.Fatalf("event.Job = %#v, want job-1", event.Job)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for job.cancelled event")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminJobsHandleCancel_QueuedSnapshotFallsBackToRunningCancellation(t *testing.T) {
|
||||
repo := &fakeAdminJobRepository{
|
||||
job: &models.AdminJob{
|
||||
ID: "job-1",
|
||||
JobType: adminjob.JobTypeLibraryRefresh,
|
||||
Status: adminjob.StatusQueued,
|
||||
CreatedByUserID: 1,
|
||||
Message: "Queued",
|
||||
RequestedAt: time.Now().UTC(),
|
||||
ProgressCurrent: 3,
|
||||
ProgressTotal: 10,
|
||||
},
|
||||
promoteToRunningOnCancelQueued: true,
|
||||
}
|
||||
cancelled := false
|
||||
registry := adminjob.NewCancelRegistry()
|
||||
unregister := registry.Register("job-1", func() {
|
||||
cancelled = true
|
||||
})
|
||||
defer unregister()
|
||||
|
||||
handler := NewAdminJobsHandler(repo, nil)
|
||||
handler.CancelRegistry = registry
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
|
||||
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusAccepted, rec.Body.String())
|
||||
}
|
||||
if !cancelled {
|
||||
t.Fatal("expected running job cancel registry to be invoked")
|
||||
}
|
||||
if repo.cancelQueuedCalls != 1 {
|
||||
t.Fatalf("CancelQueued called %d times, want 1", repo.cancelQueuedCalls)
|
||||
}
|
||||
if repo.cancelCalls != 0 {
|
||||
t.Fatalf("Cancel called %d times, want 0", repo.cancelCalls)
|
||||
}
|
||||
if repo.updateProgressCalls != 1 {
|
||||
t.Fatalf("UpdateProgress called %d times, want 1", repo.updateProgressCalls)
|
||||
}
|
||||
if repo.job.Status != adminjob.StatusRunning {
|
||||
t.Fatalf("job status = %q, want %q", repo.job.Status, adminjob.StatusRunning)
|
||||
}
|
||||
if repo.job.Message != "Cancellation requested" {
|
||||
t.Fatalf("job message = %q, want cancellation request", repo.job.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func adminJobCancelRequest(id string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/jobs/"+id+"/cancel", nil)
|
||||
routeCtx := chi.NewRouteContext()
|
||||
routeCtx.URLParams.Add("id", id)
|
||||
return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
|
||||
}
|
||||
|
||||
type fakeAdminJobRepository struct {
|
||||
job *models.AdminJob
|
||||
updateProgressCalls int
|
||||
cancelCalls int
|
||||
cancelQueuedCalls int
|
||||
promoteToRunningOnCancelQueued bool
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) List(context.Context, adminjob.ListJobsOptions) ([]*models.AdminJob, error) {
|
||||
if r.job == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return []*models.AdminJob{r.job}, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) GetByID(_ context.Context, id string) (*models.AdminJob, error) {
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return nil, adminjob.ErrJobNotFound
|
||||
}
|
||||
cp := *r.job
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) Cancel(_ context.Context, id, message string, _ time.Time) (*models.AdminJob, error) {
|
||||
r.cancelCalls++
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return nil, adminjob.ErrJobNotFound
|
||||
}
|
||||
if r.job.Status != adminjob.StatusQueued && r.job.Status != adminjob.StatusRunning {
|
||||
return nil, adminjob.ErrJobNotCancellable
|
||||
}
|
||||
cp := *r.job
|
||||
cp.Status = adminjob.StatusCancelled
|
||||
cp.Message = message
|
||||
r.job = &cp
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) CancelQueued(_ context.Context, id, message string, _ time.Time) (*models.AdminJob, error) {
|
||||
r.cancelQueuedCalls++
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return nil, adminjob.ErrJobNotFound
|
||||
}
|
||||
if r.promoteToRunningOnCancelQueued {
|
||||
cp := *r.job
|
||||
cp.Status = adminjob.StatusRunning
|
||||
r.job = &cp
|
||||
return nil, adminjob.ErrJobNotCancellable
|
||||
}
|
||||
if r.job.Status != adminjob.StatusQueued {
|
||||
return nil, adminjob.ErrJobNotCancellable
|
||||
}
|
||||
cp := *r.job
|
||||
cp.Status = adminjob.StatusCancelled
|
||||
cp.Message = message
|
||||
r.job = &cp
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) UpdateProgress(_ context.Context, id string, current, total int, message string) error {
|
||||
r.updateProgressCalls++
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return adminjob.ErrJobNotFound
|
||||
}
|
||||
if message == "" {
|
||||
return errors.New("message is required")
|
||||
}
|
||||
r.job.ProgressCurrent = current
|
||||
r.job.ProgressTotal = total
|
||||
r.job.Message = message
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,73 +1,132 @@
|
||||
package handlers
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestUpdateRequiresSessionRevocation(t *testing.T) {
|
||||
role := "admin"
|
||||
sameRole := "user"
|
||||
enabled := true
|
||||
disabled := false
|
||||
libraryIDs := []int{1, 2}
|
||||
sameLibraryIDs := []int{1}
|
||||
emptyLibraryIDs := []int{}
|
||||
var allLibraryIDs []int
|
||||
maxPlaybackQuality := "1080p"
|
||||
sameMaxPlaybackQuality := "original"
|
||||
password := "new-password"
|
||||
username := "renamed"
|
||||
maxStreams := 4
|
||||
permissions := []string{"metadata_curation"}
|
||||
samePermissions := []string{"download"}
|
||||
|
||||
current := &models.User{
|
||||
Role: "user",
|
||||
Permissions: []string{"download"},
|
||||
Enabled: false,
|
||||
LibraryIDs: []int{1},
|
||||
MaxPlaybackQuality: "original",
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
req updateUserRequest
|
||||
in models.UpdateUserInput
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "permissions set",
|
||||
req: updateUserRequest{Permissions: updateStringSliceField{Set: true, Value: []string{"metadata_curation"}}},
|
||||
in: models.UpdateUserInput{Permissions: &permissions},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "permissions unset",
|
||||
req: updateUserRequest{Permissions: updateStringSliceField{Set: false, Value: []string{"metadata_curation"}}},
|
||||
name: "permissions unchanged",
|
||||
in: models.UpdateUserInput{Permissions: &samePermissions},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "role",
|
||||
req: updateUserRequest{Role: &role},
|
||||
in: models.UpdateUserInput{Role: &role},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "role unchanged",
|
||||
in: models.UpdateUserInput{Role: &sameRole},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "enabled",
|
||||
req: updateUserRequest{Enabled: &enabled},
|
||||
in: models.UpdateUserInput{Enabled: &enabled},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "library ids",
|
||||
req: updateUserRequest{LibraryIDs: updateLibraryIDsField{Set: true, Value: libraryIDs}},
|
||||
want: true,
|
||||
name: "enabled unchanged",
|
||||
in: models.UpdateUserInput{Enabled: &disabled},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "library ids does not revoke session",
|
||||
in: models.UpdateUserInput{LibraryIDs: &libraryIDs},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "library ids unchanged",
|
||||
in: models.UpdateUserInput{LibraryIDs: &sameLibraryIDs},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "library ids nil does not revoke session",
|
||||
in: models.UpdateUserInput{LibraryIDs: &allLibraryIDs},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "max playback quality",
|
||||
req: updateUserRequest{MaxPlaybackQuality: &maxPlaybackQuality},
|
||||
in: models.UpdateUserInput{MaxPlaybackQuality: &maxPlaybackQuality},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "max playback quality unchanged",
|
||||
in: models.UpdateUserInput{MaxPlaybackQuality: &sameMaxPlaybackQuality},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "password",
|
||||
req: updateUserRequest{Password: &password},
|
||||
in: models.UpdateUserInput{Password: &password},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "non access fields",
|
||||
req: updateUserRequest{Username: &username, MaxStreams: &maxStreams},
|
||||
in: models.UpdateUserInput{Username: &username, MaxStreams: &maxStreams},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty update",
|
||||
req: updateUserRequest{},
|
||||
in: models.UpdateUserInput{},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := updateRequiresSessionRevocation(tt.req); got != tt.want {
|
||||
if got := updateRequiresSessionRevocation(current, tt.in); got != tt.want {
|
||||
t.Fatalf("updateRequiresSessionRevocation() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
unrestrictedCurrent := *current
|
||||
unrestrictedCurrent.LibraryIDs = nil
|
||||
t.Run("library ids empty does not revoke session", func(t *testing.T) {
|
||||
if got := updateRequiresSessionRevocation(&unrestrictedCurrent, models.UpdateUserInput{LibraryIDs: &emptyLibraryIDs}); got {
|
||||
t.Fatalf("updateRequiresSessionRevocation() = %v, want false", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("library ids nil unchanged", func(t *testing.T) {
|
||||
if got := updateRequiresSessionRevocation(&unrestrictedCurrent, models.UpdateUserInput{LibraryIDs: &allLibraryIDs}); got {
|
||||
t.Fatalf("updateRequiresSessionRevocation() = %v, want false", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -17,9 +17,10 @@ import (
|
||||
|
||||
// AuthHandler handles authentication-related HTTP endpoints.
|
||||
type AuthHandler struct {
|
||||
service *auth.Service
|
||||
jwt *auth.JWTService
|
||||
device *auth.DeviceLoginService
|
||||
service *auth.Service
|
||||
jwt *auth.JWTService
|
||||
device *auth.DeviceLoginService
|
||||
oauthRoutesAvailable bool
|
||||
}
|
||||
|
||||
// NewAuthHandler creates a new AuthHandler backed by the given auth, JWT,
|
||||
@@ -32,6 +33,13 @@ func NewAuthHandler(service *auth.Service, jwt *auth.JWTService, device *auth.De
|
||||
}
|
||||
}
|
||||
|
||||
// SetOAuthRoutesAvailable controls whether OAuth login providers are
|
||||
// advertised by /auth/providers. The router only mounts OAuth routes when the
|
||||
// server has enough configuration to complete the flow.
|
||||
func (h *AuthHandler) SetOAuthRoutesAvailable(available bool) {
|
||||
h.oauthRoutesAvailable = available
|
||||
}
|
||||
|
||||
// --- Request/Response types ---
|
||||
|
||||
// loginRequest represents the JSON body of a POST /auth/login request.
|
||||
@@ -181,6 +189,9 @@ func (h *AuthHandler) HandleProviders(w http.ResponseWriter, r *http.Request) {
|
||||
providers := h.service.ListProviders()
|
||||
response := make([]authProviderResponse, 0, len(providers))
|
||||
for _, provider := range providers {
|
||||
if provider.Mode == "oauth" && !h.oauthRoutesAvailable {
|
||||
continue
|
||||
}
|
||||
response = append(response, authProviderResponse{
|
||||
ID: provider.ID,
|
||||
DisplayName: provider.DisplayName,
|
||||
@@ -214,6 +225,9 @@ func (h *AuthHandler) HandleSetup(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
req.Username = auth.NormalizeUsername(req.Username)
|
||||
req.Email = auth.NormalizeEmail(req.Email)
|
||||
|
||||
if req.Username == "" || req.Email == "" || req.Password == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Username, email, and password are required")
|
||||
return
|
||||
@@ -447,6 +461,9 @@ func (h *AuthHandler) HandleSignup(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
req.Username = auth.NormalizeUsername(req.Username)
|
||||
req.Email = auth.NormalizeEmail(req.Email)
|
||||
|
||||
if req.Username == "" || req.Email == "" || req.Password == "" || req.InviteCode == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Username, email, password, and invite code are required")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type stubLoginProvider struct{}
|
||||
|
||||
func (stubLoginProvider) Authenticate(context.Context, auth.Credentials) (*models.User, error) {
|
||||
return nil, auth.ErrInvalidCredentials
|
||||
}
|
||||
|
||||
func (stubLoginProvider) ValidateSession(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func newAuthProviderHandlerForTest(oauthRoutesAvailable bool) *AuthHandler {
|
||||
service := auth.NewService(nil, nil, nil, nil, nil, nil, nil)
|
||||
provider := stubLoginProvider{}
|
||||
|
||||
service.RegisterProvider(auth.LoginProviderInfo{
|
||||
ID: "local",
|
||||
DisplayName: "Local",
|
||||
Mode: "credentials",
|
||||
Default: true,
|
||||
}, provider)
|
||||
service.RegisterProvider(auth.LoginProviderInfo{
|
||||
ID: "plugin:41:oidc",
|
||||
DisplayName: "OIDC",
|
||||
Mode: "oauth",
|
||||
InstallationID: 41,
|
||||
}, provider)
|
||||
|
||||
handler := NewAuthHandler(service, nil, nil)
|
||||
handler.SetOAuthRoutesAvailable(oauthRoutesAvailable)
|
||||
return handler
|
||||
}
|
||||
|
||||
func readProviderResponse(t *testing.T, handler *AuthHandler) []authProviderResponse {
|
||||
t.Helper()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/auth/providers", nil)
|
||||
handler.HandleProviders(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
var providers []authProviderResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&providers); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
return providers
|
||||
}
|
||||
|
||||
func TestAuthProvidersHideOAuthWhenOAuthRoutesUnavailable(t *testing.T) {
|
||||
providers := readProviderResponse(t, newAuthProviderHandlerForTest(false))
|
||||
|
||||
if len(providers) != 1 {
|
||||
t.Fatalf("provider count = %d, want 1: %#v", len(providers), providers)
|
||||
}
|
||||
if providers[0].ID != "local" {
|
||||
t.Fatalf("provider ID = %q, want local", providers[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthProvidersIncludeOAuthWhenOAuthRoutesAvailable(t *testing.T) {
|
||||
providers := readProviderResponse(t, newAuthProviderHandlerForTest(true))
|
||||
|
||||
if len(providers) != 2 {
|
||||
t.Fatalf("provider count = %d, want 2: %#v", len(providers), providers)
|
||||
}
|
||||
|
||||
var foundOAuth bool
|
||||
for _, provider := range providers {
|
||||
if provider.ID == "plugin:41:oidc" {
|
||||
foundOAuth = true
|
||||
if provider.Mode != "oauth" {
|
||||
t.Fatalf("OAuth provider mode = %q, want oauth", provider.Mode)
|
||||
}
|
||||
if provider.InstallationID != 41 {
|
||||
t.Fatalf("OAuth provider installation ID = %d, want 41", provider.InstallationID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundOAuth {
|
||||
t.Fatalf("OAuth provider missing from response: %#v", providers)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,26 +2,69 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
)
|
||||
|
||||
type calendarRepository interface {
|
||||
ListEvents(ctx context.Context, f catalog.CalendarFilter) ([]catalog.CalendarEvent, error)
|
||||
}
|
||||
|
||||
// calendarPersonalRepo resolves per-profile id-sets and watched status.
|
||||
type calendarPersonalRepo interface {
|
||||
ListFollowedItemIDs(ctx context.Context, userID int, profileID string) ([]string, error)
|
||||
ListFavoriteItemIDs(ctx context.Context, userID int, profileID string) ([]string, error)
|
||||
ListWatchlistItemIDs(ctx context.Context, userID int, profileID string) ([]string, error)
|
||||
ListWatchedItemIDs(ctx context.Context, userID int, profileID string, contentIDs []string) (map[string]bool, error)
|
||||
}
|
||||
|
||||
// calendarPopularSource reads the cached server-wide popular id-set.
|
||||
type calendarPopularSource interface {
|
||||
GetRecommendationCache(ctx context.Context, userID int, profileID, recType, sourceItemID string) ([]recommendations.ScoredItem, error)
|
||||
}
|
||||
|
||||
// calendarTrendingSource reads the external-trending snapshot.
|
||||
type calendarTrendingSource interface {
|
||||
Get(ctx context.Context, source, window string) (sections.TrendingSnapshot, bool, error)
|
||||
}
|
||||
|
||||
const (
|
||||
calendarFilterAll = "all"
|
||||
calendarFilterEverything = "everything"
|
||||
calendarFilterFollowing = "following"
|
||||
calendarFilterFavorites = "favorites"
|
||||
calendarFilterWatchlist = "watchlist"
|
||||
calendarFilterPopular = "popular"
|
||||
calendarFilterTrending = "trending"
|
||||
)
|
||||
|
||||
// The Trending preset reads this canonical external-trending snapshot
|
||||
// (see internal/sections trending snapshots).
|
||||
const (
|
||||
calendarTrendingSnapshotSource = "tmdb"
|
||||
calendarTrendingSnapshotWindow = "week"
|
||||
)
|
||||
|
||||
// CalendarHandler handles the calendar endpoint.
|
||||
type CalendarHandler struct {
|
||||
repo calendarRepository
|
||||
detailSvc *catalog.DetailService
|
||||
personal calendarPersonalRepo // nil-tolerant (per-profile presets degrade to empty)
|
||||
popular calendarPopularSource // nil when recommendations disabled
|
||||
trending calendarTrendingSource // nil when trending disabled
|
||||
}
|
||||
|
||||
// NewCalendarHandler creates a new CalendarHandler.
|
||||
func NewCalendarHandler(repo *catalog.CalendarRepository, detailSvc *catalog.DetailService) *CalendarHandler {
|
||||
return &CalendarHandler{repo: repo, detailSvc: detailSvc}
|
||||
// NewCalendarHandler creates a new CalendarHandler. The repo doubles as the
|
||||
// per-profile resolver since *catalog.CalendarRepository implements both.
|
||||
func NewCalendarHandler(repo *catalog.CalendarRepository, detailSvc *catalog.DetailService, popular calendarPopularSource, trending calendarTrendingSource) *CalendarHandler {
|
||||
return &CalendarHandler{repo: repo, detailSvc: detailSvc, personal: repo, popular: popular, trending: trending}
|
||||
}
|
||||
|
||||
// --- Response types ---
|
||||
@@ -36,8 +79,12 @@ type calendarEventResponse struct {
|
||||
EpisodeNumber *int `json:"episode_number,omitempty"`
|
||||
AirDate string `json:"air_date"`
|
||||
AirTime *string `json:"air_time,omitempty"`
|
||||
AirAt *string `json:"air_at,omitempty"`
|
||||
AirTimezone *string `json:"air_timezone,omitempty"`
|
||||
LocalAirDate string `json:"local_air_date"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
PosterThumbhash string `json:"poster_thumbhash,omitempty"`
|
||||
Watched bool `json:"watched"`
|
||||
Badges []string `json:"badges"`
|
||||
}
|
||||
|
||||
@@ -75,31 +122,39 @@ func (h *CalendarHandler) HandleGetCalendar(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
filter := q.Get("filter")
|
||||
if filter == "" {
|
||||
filter = "all"
|
||||
filter = calendarFilterAll
|
||||
}
|
||||
switch filter {
|
||||
case "all", "favorites", "watchlist":
|
||||
case calendarFilterAll, calendarFilterEverything, calendarFilterFollowing,
|
||||
calendarFilterFavorites, calendarFilterWatchlist, calendarFilterPopular, calendarFilterTrending:
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "filter must be all, favorites, or watchlist")
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "invalid filter")
|
||||
return
|
||||
}
|
||||
|
||||
af := requestAccessFilter(r)
|
||||
|
||||
if (filter == "favorites" || filter == "watchlist") && af.ProfileID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "profile required for favorites/watchlist filter")
|
||||
viewerLocation := catalog.CalendarLocation(q.Get("timezone"))
|
||||
|
||||
restrict, ids, err := h.resolveCalendarRestriction(r.Context(), filter, af)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "failed to resolve calendar filter")
|
||||
return
|
||||
}
|
||||
// A restricting preset with no ids can never match — skip the windowed query.
|
||||
if restrict && len(ids) == 0 {
|
||||
writeJSON(w, http.StatusOK, calendarResponse{Events: []calendarDayResponse{}})
|
||||
return
|
||||
}
|
||||
|
||||
cf := catalog.CalendarFilter{
|
||||
Start: start,
|
||||
End: end,
|
||||
Filter: filter,
|
||||
Start: start.AddDate(0, 0, -2),
|
||||
End: end.AddDate(0, 0, 2),
|
||||
AllowedLibraryIDs: af.AllowedLibraryIDs,
|
||||
DisabledLibraryIDs: af.DisabledLibraryIDs,
|
||||
MaxContentRating: af.MaxContentRating,
|
||||
UserID: af.UserID,
|
||||
ProfileID: af.ProfileID,
|
||||
RestrictByIDs: restrict,
|
||||
RestrictToIDs: ids,
|
||||
}
|
||||
|
||||
if v := q.Get("library_id"); v != "" {
|
||||
@@ -117,12 +172,82 @@ func (h *CalendarHandler) HandleGetCalendar(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
watched := h.resolveWatched(r.Context(), af, events)
|
||||
|
||||
// Group events by date and build response.
|
||||
days := groupEventsByDate(events, r, h.detailSvc)
|
||||
days := groupEventsByDate(events, r, h.detailSvc, start, end, viewerLocation, watched)
|
||||
writeJSON(w, http.StatusOK, calendarResponse{Events: days})
|
||||
}
|
||||
|
||||
func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSvc *catalog.DetailService) []calendarDayResponse {
|
||||
// resolveCalendarRestriction maps a preset to an id-set restriction. restrict=false
|
||||
// means no restriction (everything/all). A nil/missing source degrades that preset
|
||||
// to an empty id-set (which the caller renders as an empty calendar).
|
||||
func (h *CalendarHandler) resolveCalendarRestriction(ctx context.Context, filter string, af catalog.AccessFilter) (restrict bool, ids []string, err error) {
|
||||
switch filter {
|
||||
case calendarFilterAll, calendarFilterEverything:
|
||||
return false, nil, nil
|
||||
case calendarFilterFollowing, calendarFilterFavorites, calendarFilterWatchlist:
|
||||
if h.personal == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
switch filter {
|
||||
case calendarFilterFavorites:
|
||||
ids, err = h.personal.ListFavoriteItemIDs(ctx, af.UserID, af.ProfileID)
|
||||
case calendarFilterWatchlist:
|
||||
ids, err = h.personal.ListWatchlistItemIDs(ctx, af.UserID, af.ProfileID)
|
||||
default: // following
|
||||
ids, err = h.personal.ListFollowedItemIDs(ctx, af.UserID, af.ProfileID)
|
||||
}
|
||||
return true, ids, err
|
||||
case calendarFilterPopular:
|
||||
if h.popular == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
items, err := h.popular.GetRecommendationCache(ctx, recommendations.GlobalCacheUserID, recommendations.GlobalCacheProfileID, recommendations.RecTypePopular, "")
|
||||
if err != nil {
|
||||
return true, nil, err
|
||||
}
|
||||
ids = make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
ids = append(ids, it.MediaItemID)
|
||||
}
|
||||
return true, ids, nil
|
||||
case calendarFilterTrending:
|
||||
if h.trending == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
snap, ok, err := h.trending.Get(ctx, calendarTrendingSnapshotSource, calendarTrendingSnapshotWindow)
|
||||
if err != nil {
|
||||
return true, nil, err
|
||||
}
|
||||
if !ok {
|
||||
return true, nil, nil
|
||||
}
|
||||
return true, snap.ContentIDs, nil
|
||||
default:
|
||||
return false, nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// resolveWatched decorates events with the profile's completed status. Best-effort:
|
||||
// a lookup failure logs and returns no watched marks rather than failing the page.
|
||||
func (h *CalendarHandler) resolveWatched(ctx context.Context, af catalog.AccessFilter, events []catalog.CalendarEvent) map[string]bool {
|
||||
if h.personal == nil || len(events) == 0 {
|
||||
return map[string]bool{}
|
||||
}
|
||||
ids := make([]string, 0, len(events))
|
||||
for _, ev := range events {
|
||||
ids = append(ids, ev.ContentID)
|
||||
}
|
||||
watched, err := h.personal.ListWatchedItemIDs(ctx, af.UserID, af.ProfileID, ids)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "calendar watched overlay failed", "error", err)
|
||||
return map[string]bool{}
|
||||
}
|
||||
return watched
|
||||
}
|
||||
|
||||
func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSvc *catalog.DetailService, start, end time.Time, viewerLocation *time.Location, watched map[string]bool) []calendarDayResponse {
|
||||
if len(events) == 0 {
|
||||
return []calendarDayResponse{}
|
||||
}
|
||||
@@ -144,17 +269,78 @@ func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSv
|
||||
posterURLs = detailSvc.PresignImageURLs(r.Context(), posterPaths, "poster", "small")
|
||||
}
|
||||
|
||||
type preparedCalendarEvent struct {
|
||||
event catalog.CalendarEvent
|
||||
localDate string
|
||||
sourceDate string
|
||||
localTime time.Time
|
||||
hasTime bool
|
||||
airAtString *string
|
||||
}
|
||||
|
||||
startDate := start.Format("2006-01-02")
|
||||
endDate := end.Format("2006-01-02")
|
||||
|
||||
prepared := make([]preparedCalendarEvent, 0, len(events))
|
||||
for _, ev := range events {
|
||||
localTime, hasTime := catalog.CalendarEventLocalTime(ev.AirDate, ev.AirTime, ev.AirTimezone, viewerLocation)
|
||||
localDate := localTime.Format("2006-01-02")
|
||||
if localDate < startDate || localDate > endDate {
|
||||
continue
|
||||
}
|
||||
var airAtString *string
|
||||
if airAt := catalog.CalendarEventAirAt(ev.AirDate, ev.AirTime, ev.AirTimezone); airAt != nil {
|
||||
formatted := airAt.Format(time.RFC3339)
|
||||
airAtString = &formatted
|
||||
}
|
||||
prepared = append(prepared, preparedCalendarEvent{
|
||||
event: ev,
|
||||
localDate: localDate,
|
||||
sourceDate: ev.AirDate.Format("2006-01-02"),
|
||||
localTime: localTime,
|
||||
hasTime: hasTime,
|
||||
airAtString: airAtString,
|
||||
})
|
||||
}
|
||||
if len(prepared) == 0 {
|
||||
return []calendarDayResponse{}
|
||||
}
|
||||
|
||||
// Order each local day by the wall-clock time the viewer actually sees,
|
||||
// then place date-only entries (no air_time) after timed entries.
|
||||
sort.SliceStable(prepared, func(i, j int) bool {
|
||||
left, right := prepared[i], prepared[j]
|
||||
if left.localDate != right.localDate {
|
||||
return left.localDate < right.localDate
|
||||
}
|
||||
if left.hasTime != right.hasTime {
|
||||
return left.hasTime
|
||||
}
|
||||
if left.hasTime && !left.localTime.Equal(right.localTime) {
|
||||
return left.localTime.Before(right.localTime)
|
||||
}
|
||||
if left.event.Title != right.event.Title {
|
||||
return left.event.Title < right.event.Title
|
||||
}
|
||||
if left.event.SeasonNumber != nil && right.event.SeasonNumber != nil && *left.event.SeasonNumber != *right.event.SeasonNumber {
|
||||
return *left.event.SeasonNumber < *right.event.SeasonNumber
|
||||
}
|
||||
if left.event.EpisodeNumber != nil && right.event.EpisodeNumber != nil && *left.event.EpisodeNumber != *right.event.EpisodeNumber {
|
||||
return *left.event.EpisodeNumber < *right.event.EpisodeNumber
|
||||
}
|
||||
return left.event.ContentID < right.event.ContentID
|
||||
})
|
||||
|
||||
var days []calendarDayResponse
|
||||
var currentDay *calendarDayResponse
|
||||
|
||||
for _, ev := range events {
|
||||
dateStr := ev.AirDate.Format("2006-01-02")
|
||||
|
||||
if currentDay == nil || currentDay.Date != dateStr {
|
||||
for _, item := range prepared {
|
||||
ev := item.event
|
||||
if currentDay == nil || currentDay.Date != item.localDate {
|
||||
if currentDay != nil {
|
||||
days = append(days, *currentDay)
|
||||
}
|
||||
currentDay = &calendarDayResponse{Date: dateStr}
|
||||
currentDay = &calendarDayResponse{Date: item.localDate}
|
||||
}
|
||||
|
||||
badges := buildBadges(ev)
|
||||
@@ -167,10 +353,14 @@ func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSv
|
||||
SeriesID: ev.SeriesID,
|
||||
SeasonNumber: ev.SeasonNumber,
|
||||
EpisodeNumber: ev.EpisodeNumber,
|
||||
AirDate: dateStr,
|
||||
AirDate: item.sourceDate,
|
||||
AirTime: ev.AirTime,
|
||||
AirAt: item.airAtString,
|
||||
AirTimezone: ev.AirTimezone,
|
||||
LocalAirDate: item.localDate,
|
||||
PosterURL: posterURLs[ev.PosterPath],
|
||||
PosterThumbhash: ev.PosterThumbhash,
|
||||
Watched: watched[ev.ContentID],
|
||||
Badges: badges,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
)
|
||||
|
||||
type stubCalendarRepo struct {
|
||||
@@ -110,6 +112,62 @@ func TestHandleGetCalendar_ReturnsEmptyEvents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_ExpandsRepoRangeAndGroupsByViewerLocalDate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
airTime := "00:30"
|
||||
airTimezone := "Asia/Tokyo"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "episode-1",
|
||||
Type: "episode",
|
||||
Title: "Series",
|
||||
SeriesID: ptrString("series-1"),
|
||||
AirDate: time.Date(2026, time.January, 2, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &airTime,
|
||||
AirTimezone: &airTimezone,
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if got := repo.last.Start.Format("2006-01-02"); got != "2025-12-30" {
|
||||
t.Fatalf("repo start = %s, want expanded 2025-12-30", got)
|
||||
}
|
||||
if got := repo.last.End.Format("2006-01-02"); got != "2026-01-03" {
|
||||
t.Fatalf("repo end = %s, want expanded 2026-01-03", got)
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
if resp.Events[0].Date != "2026-01-01" {
|
||||
t.Fatalf("day = %q, want viewer-local 2026-01-01", resp.Events[0].Date)
|
||||
}
|
||||
item := resp.Events[0].Items[0]
|
||||
if item.AirDate != "2026-01-02" {
|
||||
t.Fatalf("air_date = %q, want source date 2026-01-02", item.AirDate)
|
||||
}
|
||||
if item.LocalAirDate != "2026-01-01" {
|
||||
t.Fatalf("local_air_date = %q, want 2026-01-01", item.LocalAirDate)
|
||||
}
|
||||
if item.AirAt == nil || *item.AirAt != "2026-01-01T15:30:00Z" {
|
||||
t.Fatalf("air_at = %v, want 2026-01-01T15:30:00Z", item.AirAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_GroupsEventsAndBatchResolvesCardPosters(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -189,6 +247,176 @@ func TestHandleGetCalendar_GroupsEventsAndBatchResolvesCardPosters(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_OrdersEventsWithoutTimezoneByWallClock(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
late := "22:00"
|
||||
early := "10:00"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "ep-late",
|
||||
Type: "episode",
|
||||
Title: "Aaa Show", // alphabetically first, but airs late
|
||||
SeriesID: ptrString("series-late"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &late,
|
||||
// no AirTimezone -> air_at is nil
|
||||
},
|
||||
{
|
||||
ContentID: "ep-early",
|
||||
Type: "episode",
|
||||
Title: "Zzz Show", // alphabetically last, but airs early
|
||||
SeriesID: ptrString("series-early"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &early,
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
items := resp.Events[0].Items
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items len = %d, want 2", len(items))
|
||||
}
|
||||
if items[0].ContentID != "ep-early" || items[1].ContentID != "ep-late" {
|
||||
t.Fatalf("order = [%s, %s], want [ep-early, ep-late] (ascending by air_time)",
|
||||
items[0].ContentID, items[1].ContentID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_InterleavesZonedAndUnzonedByViewerLocalTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
eightPM := "20:00"
|
||||
sixPM := "18:00"
|
||||
londonNine := "21:00"
|
||||
london := "Europe/London"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "alpha-8pm",
|
||||
Type: "episode",
|
||||
Title: "Alpha", // would sort first alphabetically
|
||||
SeriesID: ptrString("series-alpha"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &eightPM, // displays 8:00 PM ET
|
||||
},
|
||||
{
|
||||
ContentID: "bravo-4pm",
|
||||
Type: "episode",
|
||||
Title: "Bravo",
|
||||
SeriesID: ptrString("series-bravo"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &londonNine, // 21:00 GMT -> 16:00 ET (4:00 PM)
|
||||
AirTimezone: &london,
|
||||
},
|
||||
{
|
||||
ContentID: "charlie-6pm",
|
||||
Type: "episode",
|
||||
Title: "Charlie",
|
||||
SeriesID: ptrString("series-charlie"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &sixPM, // displays 6:00 PM ET
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
got := make([]string, 0, len(resp.Events[0].Items))
|
||||
for _, item := range resp.Events[0].Items {
|
||||
got = append(got, item.ContentID)
|
||||
}
|
||||
want := []string{"bravo-4pm", "charlie-6pm", "alpha-8pm"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("items = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("order = %v, want %v (ascending by viewer-local time)", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_DateOnlyEntriesSortAfterTimedEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
noon := "12:00"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "movie-1",
|
||||
Type: "movie",
|
||||
Title: "Aaa Movie", // alphabetically first, but has no air_time
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
ContentID: "ep-noon",
|
||||
Type: "episode",
|
||||
Title: "Zzz Show",
|
||||
SeriesID: ptrString("series-z"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &noon,
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
items := resp.Events[0].Items
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items len = %d, want 2", len(items))
|
||||
}
|
||||
if items[0].ContentID != "ep-noon" || items[1].ContentID != "movie-1" {
|
||||
t.Fatalf("order = [%s, %s], want [ep-noon, movie-1] (timed before date-only)",
|
||||
items[0].ContentID, items[1].ContentID)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrString(value string) *string {
|
||||
return &value
|
||||
}
|
||||
@@ -196,3 +424,162 @@ func ptrString(value string) *string {
|
||||
func ptrInt(value int) *int {
|
||||
return &value
|
||||
}
|
||||
|
||||
type stubCalendarPersonal struct {
|
||||
followed []string
|
||||
favorites []string
|
||||
watchlist []string
|
||||
watched map[string]bool
|
||||
lastWatchedIDs []string
|
||||
}
|
||||
|
||||
func (s *stubCalendarPersonal) ListFollowedItemIDs(_ context.Context, _ int, _ string) ([]string, error) {
|
||||
return s.followed, nil
|
||||
}
|
||||
func (s *stubCalendarPersonal) ListFavoriteItemIDs(_ context.Context, _ int, _ string) ([]string, error) {
|
||||
return s.favorites, nil
|
||||
}
|
||||
func (s *stubCalendarPersonal) ListWatchlistItemIDs(_ context.Context, _ int, _ string) ([]string, error) {
|
||||
return s.watchlist, nil
|
||||
}
|
||||
func (s *stubCalendarPersonal) ListWatchedItemIDs(_ context.Context, _ int, _ string, ids []string) (map[string]bool, error) {
|
||||
s.lastWatchedIDs = ids
|
||||
return s.watched, nil
|
||||
}
|
||||
|
||||
type stubPopularSource struct{ items []recommendations.ScoredItem }
|
||||
|
||||
func (s *stubPopularSource) GetRecommendationCache(_ context.Context, _ int, _, _, _ string) ([]recommendations.ScoredItem, error) {
|
||||
return s.items, nil
|
||||
}
|
||||
|
||||
type stubTrendingSource struct {
|
||||
snap sections.TrendingSnapshot
|
||||
ok bool
|
||||
}
|
||||
|
||||
func (s *stubTrendingSource) Get(_ context.Context, _, _ string) (sections.TrendingSnapshot, bool, error) {
|
||||
return s.snap, s.ok, nil
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_FollowingPassesFollowedIDs(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo, personal: &stubCalendarPersonal{followed: []string{"s1", "s2"}}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=following", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if !repo.last.RestrictByIDs {
|
||||
t.Fatalf("expected RestrictByIDs true")
|
||||
}
|
||||
if got := repo.last.RestrictToIDs; len(got) != 2 || got[0] != "s1" || got[1] != "s2" {
|
||||
t.Fatalf("RestrictToIDs = %v, want [s1 s2]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_PopularReadsCache(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{
|
||||
repo: repo,
|
||||
popular: &stubPopularSource{items: []recommendations.ScoredItem{{MediaItemID: "p1"}, {MediaItemID: "p2"}}},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=popular", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if !repo.last.RestrictByIDs || len(repo.last.RestrictToIDs) != 2 || repo.last.RestrictToIDs[0] != "p1" {
|
||||
t.Fatalf("RestrictToIDs = %v, want [p1 p2]", repo.last.RestrictToIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_TrendingReadsSnapshot(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{
|
||||
repo: repo,
|
||||
trending: &stubTrendingSource{ok: true, snap: sections.TrendingSnapshot{ContentIDs: []string{"t1"}}},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=trending", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if !repo.last.RestrictByIDs || len(repo.last.RestrictToIDs) != 1 || repo.last.RestrictToIDs[0] != "t1" {
|
||||
t.Fatalf("RestrictToIDs = %v, want [t1]", repo.last.RestrictToIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_EverythingHasNoRestriction(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=everything", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if repo.last.RestrictByIDs {
|
||||
t.Fatalf("expected no restriction for everything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_RejectsUnknownFilter(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=bogus", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("expected repo not called, got %d", repo.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_EmptyFollowedShortCircuits(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo, personal: &stubCalendarPersonal{followed: nil}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=following", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("expected ListEvents skipped for empty followed set, got %d calls", repo.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_MarksWatchedItems(t *testing.T) {
|
||||
repo := &stubCalendarRepo{events: []catalog.CalendarEvent{
|
||||
{ContentID: "ep-1", Type: "episode", Title: "Show", AirDate: time.Date(2026, time.April, 8, 0, 0, 0, 0, time.UTC)},
|
||||
{ContentID: "ep-2", Type: "episode", Title: "Show", AirDate: time.Date(2026, time.April, 8, 0, 0, 0, 0, time.UTC)},
|
||||
}}
|
||||
handler := &CalendarHandler{repo: repo, personal: &stubCalendarPersonal{watched: map[string]bool{"ep-1": true}}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=everything", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
watched := map[string]bool{}
|
||||
for _, day := range resp.Events {
|
||||
for _, item := range day.Items {
|
||||
watched[item.ContentID] = item.Watched
|
||||
}
|
||||
}
|
||||
if !watched["ep-1"] || watched["ep-2"] {
|
||||
t.Fatalf("watched flags = %v, want ep-1 true / ep-2 false", watched)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,6 +80,17 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request
|
||||
overlaySummaries := h.itemsH.listOverlaySummaries(r.Context(), result.Items, h.itemsH.accessFilter(r))
|
||||
userStates := h.itemsH.listItemUserStates(r, result.Items)
|
||||
episodeMetadata := h.itemsH.listEpisodeBrowseMetadata(r.Context(), result.Items)
|
||||
sortField := catalog.NormalizeQuerySort(req.Query.Sort).Field
|
||||
store, profileID, _ := h.itemsH.userStoreForRequest(r)
|
||||
sortMetrics := h.itemsH.listSortMetrics(
|
||||
r.Context(),
|
||||
result.Items,
|
||||
sortField,
|
||||
h.itemsH.accessFilter(r),
|
||||
overlaySummaries,
|
||||
store,
|
||||
profileID,
|
||||
)
|
||||
items := make([]itemListResponse, 0, len(result.Items))
|
||||
for _, item := range result.Items {
|
||||
resp := h.itemsH.toItemListResponseWithOverlay(
|
||||
@@ -93,6 +104,7 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request
|
||||
resp.SeasonNumber = meta.SeasonNumber
|
||||
resp.EpisodeNumber = meta.EpisodeNumber
|
||||
}
|
||||
resp.SortMetrics = sortMetrics[item.ContentID]
|
||||
items = append(items, resp)
|
||||
}
|
||||
|
||||
|
||||
@@ -23,9 +23,10 @@ func publishEventMetadataUpdate(ctx context.Context, hub *evt.Hub, libraryID int
|
||||
if hub == nil {
|
||||
return
|
||||
}
|
||||
if err := hub.PublishJSON(ctx, evt.ChannelCatalog, "metadata.updated", map[string]any{
|
||||
if err := hub.PublishJSON(ctx, evt.ChannelCatalog, "catalog.item.changed", map[string]any{
|
||||
"library_id": libraryID,
|
||||
"content_id": contentID,
|
||||
"change": "metadata_updated",
|
||||
}, evt.PublishOptions{}); err != nil {
|
||||
slog.Warn("events: failed to publish metadata update", "library_id", libraryID, "content_id", contentID, "error", err)
|
||||
}
|
||||
|
||||
@@ -216,7 +216,9 @@ func (h *PersonalDataHandler) HandleAddFavorite(w http.ResponseWriter, r *http.R
|
||||
|
||||
h.dispatchLocalFavoriteEvent(r.Context(), watchsync.LocalFavoriteEventAdded, userID, profileID, itemID)
|
||||
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", "favorite.updated")
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", userStateEventState{
|
||||
IsFavorite: boolPtr(true),
|
||||
})
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -244,7 +246,9 @@ func (h *PersonalDataHandler) HandleRemoveFavorite(w http.ResponseWriter, r *htt
|
||||
|
||||
h.dispatchLocalFavoriteEvent(r.Context(), watchsync.LocalFavoriteEventRemoved, userID, profileID, itemID)
|
||||
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", "favorite.updated")
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "favorite", userStateEventState{
|
||||
IsFavorite: boolPtr(false),
|
||||
})
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -366,7 +370,9 @@ func (h *PersonalDataHandler) HandleAddToWatchlist(w http.ResponseWriter, r *htt
|
||||
}
|
||||
|
||||
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", "watchlist.updated")
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", userStateEventState{
|
||||
InWatchlist: boolPtr(true),
|
||||
})
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -393,7 +399,9 @@ func (h *PersonalDataHandler) HandleRemoveFromWatchlist(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", "watchlist.updated")
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, itemID, "", "watchlist", userStateEventState{
|
||||
InWatchlist: boolPtr(false),
|
||||
})
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -492,7 +500,7 @@ func (h *PersonalDataHandler) HandleRemoveHistory(w http.ResponseWriter, r *http
|
||||
mediaItemID,
|
||||
"",
|
||||
"history",
|
||||
"history.updated",
|
||||
userStateEventState{},
|
||||
)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
type HomeDismissalHandler struct {
|
||||
storeProvider userstore.UserStoreProvider
|
||||
EventsHub *evt.Hub
|
||||
}
|
||||
|
||||
type upsertHomeDismissalRequest struct {
|
||||
@@ -76,6 +78,16 @@ func (h *HomeDismissalHandler) HandleUpsertDismissal(w http.ResponseWriter, r *h
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save dismissal")
|
||||
return
|
||||
}
|
||||
publishUserStateEvent(
|
||||
r.Context(),
|
||||
h.EventsHub,
|
||||
userID,
|
||||
profileID,
|
||||
itemID,
|
||||
req.SeriesID,
|
||||
"home_dismissal",
|
||||
userStateEventState{},
|
||||
)
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -104,6 +116,16 @@ func (h *HomeDismissalHandler) HandleDeleteDismissal(w http.ResponseWriter, r *h
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete dismissal")
|
||||
return
|
||||
}
|
||||
publishUserStateEvent(
|
||||
r.Context(),
|
||||
h.EventsHub,
|
||||
userID,
|
||||
profileID,
|
||||
itemID,
|
||||
"",
|
||||
"home_dismissal",
|
||||
userStateEventState{},
|
||||
)
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
+254
-22
@@ -34,6 +34,10 @@ type EpisodeFileProvider interface {
|
||||
ListByContentIDs(ctx context.Context, contentIDs []string) (map[string][]*models.MediaFile, error)
|
||||
}
|
||||
|
||||
type batchEpisodeFileProvider interface {
|
||||
ListByEpisodeIDs(ctx context.Context, episodeIDs []string) (map[string][]*models.MediaFile, error)
|
||||
}
|
||||
|
||||
type MetadataRefreshRequester interface {
|
||||
RequestStaleMetadataRefresh(ctx context.Context, targetType, contentID string) error
|
||||
}
|
||||
@@ -209,9 +213,20 @@ type itemListResponse struct {
|
||||
LastAirDate *string `json:"last_air_date,omitempty"`
|
||||
AddedAt *time.Time `json:"added_at,omitempty"`
|
||||
OverlaySummary *models.OverlaySummary `json:"overlay_summary,omitempty"`
|
||||
SortMetrics *sortMetricsResponse `json:"sort_metrics,omitempty"`
|
||||
UserState *itemUserStateResponse `json:"user_state,omitempty"`
|
||||
}
|
||||
|
||||
type sortMetricsResponse struct {
|
||||
ReleaseDate *string `json:"release_date,omitempty"`
|
||||
RuntimeMinutes *int `json:"runtime_minutes,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
BitrateKbps *int `json:"bitrate_kbps,omitempty"`
|
||||
ProgressRatio *float64 `json:"progress_ratio,omitempty"`
|
||||
ViewedAt string `json:"viewed_at,omitempty"`
|
||||
PlayCount *int `json:"play_count,omitempty"`
|
||||
}
|
||||
|
||||
// browseResponse is the paginated response for the /items endpoint.
|
||||
type browseResponse struct {
|
||||
Total int `json:"total"`
|
||||
@@ -461,7 +476,9 @@ func (h *ItemsHandler) handleSetWatchedState(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
triggerProfileRefresh(r.Context(), h.profileStaler, h.profileRefreshRequester, userID, profileID)
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, id, "", "watched", "watched.updated")
|
||||
publishUserStateEvent(r.Context(), h.EventsHub, userID, profileID, id, "", "watched", userStateEventState{
|
||||
Played: boolPtr(played),
|
||||
})
|
||||
|
||||
writeJSON(w, http.StatusOK, watchedStateResponse{
|
||||
ContentID: id,
|
||||
@@ -817,6 +834,147 @@ func (h *ItemsHandler) listOverlaySummaries(ctx context.Context, items []*models
|
||||
return summaries
|
||||
}
|
||||
|
||||
groupedFiles := h.listBrowseItemFiles(ctx, items, filter)
|
||||
for contentID, files := range groupedFiles {
|
||||
if summary := overlays.BuildSummary(files); summary != nil {
|
||||
summaries[contentID] = summary
|
||||
}
|
||||
}
|
||||
return summaries
|
||||
}
|
||||
|
||||
func (h *ItemsHandler) listSortMetrics(
|
||||
ctx context.Context,
|
||||
items []*models.MediaItem,
|
||||
sortField string,
|
||||
filter catalog.AccessFilter,
|
||||
overlaySummaries map[string]*models.OverlaySummary,
|
||||
store userstore.UserStore,
|
||||
profileID string,
|
||||
) map[string]*sortMetricsResponse {
|
||||
metrics := make(map[string]*sortMetricsResponse, len(items))
|
||||
switch sortField {
|
||||
case "release_date":
|
||||
for _, item := range items {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
releaseDate := firstNonBlankPtr(item.ReleaseDate, item.FirstAirDate)
|
||||
if releaseDate != nil {
|
||||
metrics[item.ContentID] = &sortMetricsResponse{ReleaseDate: releaseDate}
|
||||
}
|
||||
}
|
||||
case "runtime":
|
||||
for _, item := range items {
|
||||
if item == nil || item.Runtime <= 0 {
|
||||
continue
|
||||
}
|
||||
runtimeMinutes := item.Runtime
|
||||
metrics[item.ContentID] = &sortMetricsResponse{RuntimeMinutes: &runtimeMinutes}
|
||||
}
|
||||
case "resolution":
|
||||
for _, item := range items {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
if summary := overlaySummaries[item.ContentID]; summary != nil && strings.TrimSpace(summary.Resolution) != "" {
|
||||
metrics[item.ContentID] = &sortMetricsResponse{Resolution: summary.Resolution}
|
||||
}
|
||||
}
|
||||
case "bitrate":
|
||||
groupedFiles := h.listBrowseItemFiles(ctx, items, filter)
|
||||
for contentID, files := range groupedFiles {
|
||||
if bitrate := maxFileBitrate(files); bitrate > 0 {
|
||||
value := bitrate
|
||||
metrics[contentID] = &sortMetricsResponse{BitrateKbps: &value}
|
||||
}
|
||||
}
|
||||
case "progress", "date_viewed", "plays":
|
||||
h.listUserSortMetrics(ctx, items, sortField, store, profileID, metrics)
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
func (h *ItemsHandler) listUserSortMetrics(
|
||||
ctx context.Context,
|
||||
items []*models.MediaItem,
|
||||
sortField string,
|
||||
store userstore.UserStore,
|
||||
profileID string,
|
||||
metrics map[string]*sortMetricsResponse,
|
||||
) {
|
||||
if store == nil || profileID == "" || len(items) == 0 {
|
||||
return
|
||||
}
|
||||
contentIDs := uniqueItemContentIDs(items)
|
||||
if len(contentIDs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
progressMap, err := store.ListProgressByMediaItems(ctx, profileID, contentIDs)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch sortField {
|
||||
case "progress":
|
||||
for _, item := range items {
|
||||
if item == nil || item.ContentID == "" {
|
||||
continue
|
||||
}
|
||||
progress, ok := progressMap[item.ContentID]
|
||||
if !ok || progress.Completed || progress.PositionSeconds <= 0 || progress.DurationSeconds <= 0 {
|
||||
continue
|
||||
}
|
||||
ratio := progress.PositionSeconds / progress.DurationSeconds
|
||||
metrics[item.ContentID] = &sortMetricsResponse{ProgressRatio: &ratio}
|
||||
}
|
||||
case "date_viewed", "plays":
|
||||
history, err := listCompletedHistoryForItems(ctx, store, profileID, contentIDs)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
historyCounts := make(map[string]int, len(contentIDs))
|
||||
historyViewedAt := make(map[string]string, len(contentIDs))
|
||||
for _, entry := range history {
|
||||
if entry.MediaItemID == "" {
|
||||
continue
|
||||
}
|
||||
historyCounts[entry.MediaItemID]++
|
||||
if entry.WatchedAt > historyViewedAt[entry.MediaItemID] {
|
||||
historyViewedAt[entry.MediaItemID] = entry.WatchedAt
|
||||
}
|
||||
}
|
||||
for _, item := range items {
|
||||
if item == nil || item.ContentID == "" {
|
||||
continue
|
||||
}
|
||||
resp := &sortMetricsResponse{}
|
||||
if sortField == "date_viewed" {
|
||||
viewedAt := historyViewedAt[item.ContentID]
|
||||
if progress, ok := progressMap[item.ContentID]; ok && progress.Completed && progress.UpdatedAt > viewedAt {
|
||||
viewedAt = progress.UpdatedAt
|
||||
}
|
||||
if viewedAt == "" {
|
||||
continue
|
||||
}
|
||||
resp.ViewedAt = viewedAt
|
||||
} else {
|
||||
playCount := historyCounts[item.ContentID]
|
||||
if progress, ok := progressMap[item.ContentID]; ok && progress.Completed && playCount < 1 {
|
||||
playCount = 1
|
||||
}
|
||||
if playCount <= 0 {
|
||||
continue
|
||||
}
|
||||
resp.PlayCount = &playCount
|
||||
}
|
||||
metrics[item.ContentID] = resp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func uniqueItemContentIDs(items []*models.MediaItem) []string {
|
||||
contentIDs := make([]string, 0, len(items))
|
||||
seen := make(map[string]struct{}, len(items))
|
||||
for _, item := range items {
|
||||
@@ -829,21 +987,100 @@ func (h *ItemsHandler) listOverlaySummaries(ctx context.Context, items []*models
|
||||
seen[item.ContentID] = struct{}{}
|
||||
contentIDs = append(contentIDs, item.ContentID)
|
||||
}
|
||||
if len(contentIDs) == 0 {
|
||||
return summaries
|
||||
}
|
||||
return contentIDs
|
||||
}
|
||||
|
||||
groupedFiles, err := h.fileRepo.ListByContentIDs(ctx, contentIDs)
|
||||
if err != nil {
|
||||
return summaries
|
||||
}
|
||||
for contentID, files := range groupedFiles {
|
||||
files = catalog.FilterMediaFilesByAccess(files, filter)
|
||||
if summary := overlays.BuildSummary(files); summary != nil {
|
||||
summaries[contentID] = summary
|
||||
func listCompletedHistoryForItems(
|
||||
ctx context.Context,
|
||||
store userstore.UserStore,
|
||||
profileID string,
|
||||
contentIDs []string,
|
||||
) ([]userstore.WatchHistoryEntry, error) {
|
||||
const pageSize = 500
|
||||
var all []userstore.WatchHistoryEntry
|
||||
for offset := 0; ; offset += pageSize {
|
||||
page, err := store.ListCompletedHistory(ctx, userstore.CompletedHistoryQuery{
|
||||
ProfileID: profileID,
|
||||
MediaItemIDs: contentIDs,
|
||||
Limit: pageSize,
|
||||
Offset: offset,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, page...)
|
||||
if len(page) < pageSize {
|
||||
return all, nil
|
||||
}
|
||||
}
|
||||
return summaries
|
||||
}
|
||||
|
||||
func (h *ItemsHandler) listBrowseItemFiles(ctx context.Context, items []*models.MediaItem, filter catalog.AccessFilter) map[string][]*models.MediaFile {
|
||||
grouped := make(map[string][]*models.MediaFile, len(items))
|
||||
if h.fileRepo == nil || len(items) == 0 {
|
||||
return grouped
|
||||
}
|
||||
|
||||
contentIDs := make([]string, 0, len(items))
|
||||
episodeIDs := make([]string, 0)
|
||||
seen := make(map[string]struct{}, len(items))
|
||||
for _, item := range items {
|
||||
if item == nil || item.ContentID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[item.ContentID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[item.ContentID] = struct{}{}
|
||||
if item.Type == "episode" {
|
||||
episodeIDs = append(episodeIDs, item.ContentID)
|
||||
} else {
|
||||
contentIDs = append(contentIDs, item.ContentID)
|
||||
}
|
||||
}
|
||||
|
||||
if len(contentIDs) > 0 {
|
||||
contentFiles, err := h.fileRepo.ListByContentIDs(ctx, contentIDs)
|
||||
if err == nil {
|
||||
for contentID, files := range contentFiles {
|
||||
grouped[contentID] = catalog.FilterMediaFilesByAccess(files, filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(episodeIDs) > 0 {
|
||||
if batchProvider, ok := h.fileRepo.(batchEpisodeFileProvider); ok {
|
||||
episodeFiles, err := batchProvider.ListByEpisodeIDs(ctx, episodeIDs)
|
||||
if err == nil {
|
||||
for episodeID, files := range episodeFiles {
|
||||
grouped[episodeID] = catalog.FilterMediaFilesByAccess(files, filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return grouped
|
||||
}
|
||||
|
||||
func firstNonBlankPtr(values ...*string) *string {
|
||||
for _, value := range values {
|
||||
if value == nil || strings.TrimSpace(*value) == "" {
|
||||
continue
|
||||
}
|
||||
copyValue := *value
|
||||
return ©Value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func maxFileBitrate(files []*models.MediaFile) int {
|
||||
maxBitrate := 0
|
||||
for _, file := range files {
|
||||
if file != nil && file.Bitrate > maxBitrate {
|
||||
maxBitrate = file.Bitrate
|
||||
}
|
||||
}
|
||||
return maxBitrate
|
||||
}
|
||||
|
||||
// toSeasonResponse converts a Season model to an API response.
|
||||
@@ -996,16 +1233,11 @@ func featuredPosterPath(path string) string {
|
||||
}
|
||||
|
||||
// featuredBackdropPath converts an S3 backdrop path from original to w1920 for
|
||||
// featured/hero contexts (displayed at full viewport width).
|
||||
// Full URLs (TMDB/TVDB) and plugin-prefixed paths are returned as-is.
|
||||
// featured/hero contexts (displayed at full viewport width). Episode stills
|
||||
// used as backdrops lack a w1920 variant and clamp to their largest cached
|
||||
// size. Full URLs (TMDB/TVDB) and plugin-prefixed paths are returned as-is.
|
||||
func featuredBackdropPath(path string) string {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return path
|
||||
}
|
||||
if strings.Contains(path, "://") {
|
||||
return path
|
||||
}
|
||||
return strings.Replace(path, "/original.", "/w1920.", 1)
|
||||
return catalog.BackdropVariantPath(path, "w1920")
|
||||
}
|
||||
|
||||
// presignURL resolves an image path to a usable URL, delegating to the
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
func TestListSortMetricsIncludesUserStateSorts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newProfileTestStore(t)
|
||||
items := []*models.MediaItem{
|
||||
{ContentID: "movie-progress", Type: "movie", Title: "Progress"},
|
||||
{ContentID: "movie-viewed", Type: "movie", Title: "Viewed"},
|
||||
{ContentID: "movie-plays", Type: "movie", Title: "Plays"},
|
||||
}
|
||||
handler := &ItemsHandler{}
|
||||
|
||||
if err := store.SetProgress(ctx, "profile-1", "movie-progress", 900, 3600, userstore.ProgressThresholds{}); err != nil {
|
||||
t.Fatalf("SetProgress(progress): %v", err)
|
||||
}
|
||||
completedAt := time.Date(2026, 5, 29, 18, 30, 0, 0, time.UTC)
|
||||
if err := store.SetProgressAt(ctx, "profile-1", "movie-viewed", 3600, 3600, true, completedAt); err != nil {
|
||||
t.Fatalf("SetProgressAt(viewed): %v", err)
|
||||
}
|
||||
for _, watchedAt := range []string{"2026-05-27T10:00:00Z", "2026-05-28T10:00:00Z"} {
|
||||
if err := store.AddHistory(ctx, userstore.WatchHistoryEntry{
|
||||
ProfileID: "profile-1",
|
||||
MediaItemID: "movie-plays",
|
||||
WatchedAt: watchedAt,
|
||||
DurationSeconds: 3600,
|
||||
Completed: true,
|
||||
Source: userstore.WatchHistorySourcePlayback,
|
||||
}); err != nil {
|
||||
t.Fatalf("AddHistory(%s): %v", watchedAt, err)
|
||||
}
|
||||
}
|
||||
|
||||
progressMetrics := handler.listSortMetrics(ctx, items, "progress", catalog.AccessFilter{}, nil, store, "profile-1")
|
||||
progress := progressMetrics["movie-progress"]
|
||||
if progress == nil || progress.ProgressRatio == nil || *progress.ProgressRatio != 0.25 {
|
||||
t.Fatalf("progress metrics = %#v", progress)
|
||||
}
|
||||
|
||||
viewedMetrics := handler.listSortMetrics(ctx, items, "date_viewed", catalog.AccessFilter{}, nil, store, "profile-1")
|
||||
if got := viewedMetrics["movie-viewed"]; got == nil || got.ViewedAt != "2026-05-29T18:30:00Z" {
|
||||
t.Fatalf("viewed metrics = %#v", got)
|
||||
}
|
||||
|
||||
playsMetrics := handler.listSortMetrics(ctx, items, "plays", catalog.AccessFilter{}, nil, store, "profile-1")
|
||||
plays := playsMetrics["movie-plays"]
|
||||
if plays == nil || plays.PlayCount == nil || *plays.PlayCount != 2 {
|
||||
t.Fatalf("plays metrics = %#v", plays)
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,10 @@ type LibraryHandler struct {
|
||||
PluginInstallations pluginInstallationLister
|
||||
SkippedRootRepo *metadata.SkippedRootRepository
|
||||
StaleIDRepo *metadata.StaleMediaIDRepository
|
||||
MovieMatchQueueRepo libraryMovieMatchQueue
|
||||
SeriesMatchQueueRepo librarySeriesMatchQueue
|
||||
RawMatchBacklogRepo libraryRawMatchBacklog
|
||||
TVSeriesRootQueue bool
|
||||
ScannedGroupRepo *scanner.ScannedGroupRepository
|
||||
GroupOverrideRepo *scanner.MediaGroupOverrideRepository
|
||||
ObservedLocationRepo *scanner.ObservedLocationRepository
|
||||
@@ -94,6 +98,27 @@ type libraryScanQueuer interface {
|
||||
CancelByLibrary(ctx context.Context, libraryID int) (int, error)
|
||||
}
|
||||
|
||||
type libraryMovieMatchQueue interface {
|
||||
SyncForFolder(ctx context.Context, folderID int) error
|
||||
DeleteByFolder(ctx context.Context, folderID int) (int, error)
|
||||
CountByFolder(ctx context.Context, folderID int) (int, error)
|
||||
ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.MovieMatchQueueEntry, int, error)
|
||||
}
|
||||
|
||||
type librarySeriesMatchQueue interface {
|
||||
SyncForFolder(ctx context.Context, folderID int) error
|
||||
DeleteByFolder(ctx context.Context, folderID int) (int, error)
|
||||
CountByFolder(ctx context.Context, folderID int) (int, error)
|
||||
ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.SeriesRootMatchQueueEntry, int, error)
|
||||
}
|
||||
|
||||
type libraryRawMatchBacklog interface {
|
||||
CountUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error)
|
||||
ListUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode, limit int, offset int) ([]*models.MediaFile, int, error)
|
||||
SuppressUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error)
|
||||
RetryUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error)
|
||||
}
|
||||
|
||||
// NewLibraryHandler creates a new LibraryHandler backed by the given folder
|
||||
// repository and ingest executor. The ingester may be nil if scan endpoints are not needed.
|
||||
func NewLibraryHandler(
|
||||
@@ -1061,6 +1086,9 @@ func (h *LibraryHandler) recordAcceptedScan(scanID string, target *scantrigger.T
|
||||
Trigger: target.Trigger,
|
||||
Status: "accepted",
|
||||
})
|
||||
if run, ok := h.ScanRegistry.Get(scanID); ok {
|
||||
h.publishScanEvent(context.Background(), "scan.accepted", run)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) markScanRunning(scanID string) {
|
||||
@@ -1279,6 +1307,371 @@ type refreshLibraryMetadataRequest struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
type libraryMetadataMatchQueueStatusResponse struct {
|
||||
LibraryID int `json:"library_id"`
|
||||
MovieCount int `json:"movie_count"`
|
||||
SeriesCount int `json:"series_count"`
|
||||
RawFileCount int `json:"raw_file_count"`
|
||||
TotalCount int `json:"total_count"`
|
||||
}
|
||||
|
||||
type libraryMetadataMatchQueueActionResponse struct {
|
||||
Status string `json:"status"`
|
||||
LibraryID int `json:"library_id"`
|
||||
MovieCancelled int `json:"movie_cancelled,omitempty"`
|
||||
SeriesCancelled int `json:"series_cancelled,omitempty"`
|
||||
RawFileCancelled int `json:"raw_file_cancelled,omitempty"`
|
||||
RawFileRetried int `json:"raw_file_retried,omitempty"`
|
||||
TotalCancelled int `json:"total_cancelled,omitempty"`
|
||||
Queue libraryMetadataMatchQueueStatusResponse `json:"queue"`
|
||||
}
|
||||
|
||||
type libraryMetadataMatchQueueDetailResponse struct {
|
||||
libraryMetadataMatchQueueStatusResponse
|
||||
Movies []libraryMovieMatchQueueEntryResponse `json:"movies"`
|
||||
Series []librarySeriesMatchQueueEntryResponse `json:"series"`
|
||||
RawFiles []libraryRawMatchBacklogEntryResponse `json:"raw_files"`
|
||||
}
|
||||
|
||||
type libraryMovieMatchQueueEntryResponse struct {
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
MediaFolderID int `json:"media_folder_id"`
|
||||
FilePath string `json:"file_path"`
|
||||
FirstQueuedAt time.Time `json:"first_queued_at"`
|
||||
AvailableAt time.Time `json:"available_at"`
|
||||
LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type librarySeriesMatchQueueEntryResponse struct {
|
||||
MediaFolderID int `json:"media_folder_id"`
|
||||
ObservedRootPath string `json:"observed_root_path"`
|
||||
FirstQueuedAt time.Time `json:"first_queued_at"`
|
||||
AvailableAt time.Time `json:"available_at"`
|
||||
LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type libraryRawMatchBacklogEntryResponse struct {
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
MediaFolderID int `json:"media_folder_id"`
|
||||
FilePath string `json:"file_path"`
|
||||
BaseTitle string `json:"base_title,omitempty"`
|
||||
BaseYear int `json:"base_year,omitempty"`
|
||||
BaseType string `json:"base_type,omitempty"`
|
||||
LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) HandleListMetadataMatchQueues(w http.ResponseWriter, r *http.Request) {
|
||||
if h.folderRepo == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Library repository is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
folders, err := h.folderRepo.List(r.Context())
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to list libraries", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queues")
|
||||
return
|
||||
}
|
||||
|
||||
resp := make([]libraryMetadataMatchQueueStatusResponse, 0, len(folders))
|
||||
for _, folder := range folders {
|
||||
if folder == nil {
|
||||
continue
|
||||
}
|
||||
status, err := h.metadataMatchQueueStatus(r.Context(), folder.ID)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to load queue status", "library_id", folder.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
|
||||
return
|
||||
}
|
||||
resp = append(resp, status)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) HandleGetMetadataMatchQueue(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.metadataMatchBacklogConfigured() {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Metadata matcher backlog is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
id, err := parseIDParam(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid library ID")
|
||||
return
|
||||
}
|
||||
if _, err := h.folderRepo.GetByID(r.Context(), id); err != nil {
|
||||
if errors.Is(err, catalog.ErrFolderNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Library not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch library")
|
||||
return
|
||||
}
|
||||
|
||||
limit := 10
|
||||
if value := strings.TrimSpace(r.URL.Query().Get("limit")); value != "" {
|
||||
if parsed, parseErr := strconv.Atoi(value); parseErr == nil && parsed > 0 && parsed <= 50 {
|
||||
limit = parsed
|
||||
}
|
||||
}
|
||||
|
||||
status, err := h.metadataMatchQueueStatus(r.Context(), id)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to load queue status", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
|
||||
return
|
||||
}
|
||||
|
||||
resp := libraryMetadataMatchQueueDetailResponse{
|
||||
libraryMetadataMatchQueueStatusResponse: status,
|
||||
Movies: []libraryMovieMatchQueueEntryResponse{},
|
||||
Series: []librarySeriesMatchQueueEntryResponse{},
|
||||
RawFiles: []libraryRawMatchBacklogEntryResponse{},
|
||||
}
|
||||
if h.MovieMatchQueueRepo != nil {
|
||||
movies, _, err := h.MovieMatchQueueRepo.ListByFolder(r.Context(), id, limit, 0)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to list movie queue", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queue")
|
||||
return
|
||||
}
|
||||
for _, entry := range movies {
|
||||
resp.Movies = append(resp.Movies, libraryMovieMatchQueueEntryResponse{
|
||||
MediaFileID: entry.MediaFileID,
|
||||
MediaFolderID: entry.MediaFolderID,
|
||||
FilePath: entry.FilePath,
|
||||
FirstQueuedAt: entry.FirstQueuedAt,
|
||||
AvailableAt: entry.AvailableAt,
|
||||
LastAttemptedAt: entry.LastAttemptedAt,
|
||||
AttemptCount: entry.AttemptCount,
|
||||
LastError: entry.LastError,
|
||||
UpdatedAt: entry.UpdatedAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
if h.SeriesMatchQueueRepo != nil {
|
||||
series, _, err := h.SeriesMatchQueueRepo.ListByFolder(r.Context(), id, limit, 0)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to list series queue", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queue")
|
||||
return
|
||||
}
|
||||
for _, entry := range series {
|
||||
resp.Series = append(resp.Series, librarySeriesMatchQueueEntryResponse{
|
||||
MediaFolderID: entry.MediaFolderID,
|
||||
ObservedRootPath: entry.ObservedRootPath,
|
||||
FirstQueuedAt: entry.FirstQueuedAt,
|
||||
AvailableAt: entry.AvailableAt,
|
||||
LastAttemptedAt: entry.LastAttemptedAt,
|
||||
AttemptCount: entry.AttemptCount,
|
||||
LastError: entry.LastError,
|
||||
UpdatedAt: entry.UpdatedAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
if h.RawMatchBacklogRepo != nil {
|
||||
rawFiles, _, err := h.RawMatchBacklogRepo.ListUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode(), limit, 0)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to list raw backlog", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher backlog")
|
||||
return
|
||||
}
|
||||
for _, file := range rawFiles {
|
||||
if file == nil {
|
||||
continue
|
||||
}
|
||||
resp.RawFiles = append(resp.RawFiles, libraryRawMatchBacklogEntryResponse{
|
||||
MediaFileID: file.ID,
|
||||
MediaFolderID: file.MediaFolderID,
|
||||
FilePath: file.FilePath,
|
||||
BaseTitle: file.BaseTitle,
|
||||
BaseYear: file.BaseYear,
|
||||
BaseType: file.BaseType,
|
||||
LastAttemptedAt: file.MatchAttemptedAt,
|
||||
CreatedAt: file.CreatedAt,
|
||||
UpdatedAt: file.UpdatedAt,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) HandleRetryMetadataMatchQueue(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.metadataMatchBacklogConfigured() {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Metadata matcher backlog is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
id, err := parseIDParam(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid library ID")
|
||||
return
|
||||
}
|
||||
if _, err := h.folderRepo.GetByID(r.Context(), id); err != nil {
|
||||
if errors.Is(err, catalog.ErrFolderNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Library not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch library")
|
||||
return
|
||||
}
|
||||
|
||||
if h.SeriesMatchQueueRepo != nil {
|
||||
if err := h.SeriesMatchQueueRepo.SyncForFolder(r.Context(), id); err != nil {
|
||||
slog.Error("metadata queue: failed to retry series queue", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher")
|
||||
return
|
||||
}
|
||||
}
|
||||
if h.MovieMatchQueueRepo != nil {
|
||||
if err := h.MovieMatchQueueRepo.SyncForFolder(r.Context(), id); err != nil {
|
||||
slog.Error("metadata queue: failed to retry movie queue", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher")
|
||||
return
|
||||
}
|
||||
}
|
||||
rawFileRetried := 0
|
||||
if h.RawMatchBacklogRepo != nil {
|
||||
rawFileRetried, err = h.RawMatchBacklogRepo.RetryUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode())
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to retry raw backlog", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
status, err := h.metadataMatchQueueStatus(r.Context(), id)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to load retried queue status", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, libraryMetadataMatchQueueActionResponse{
|
||||
Status: "queued",
|
||||
LibraryID: id,
|
||||
RawFileRetried: rawFileRetried,
|
||||
Queue: status,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) HandleCancelMetadataMatchQueue(w http.ResponseWriter, r *http.Request) {
|
||||
if !h.metadataMatchBacklogConfigured() {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Metadata matcher backlog is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
id, err := parseIDParam(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid library ID")
|
||||
return
|
||||
}
|
||||
if _, err := h.folderRepo.GetByID(r.Context(), id); err != nil {
|
||||
if errors.Is(err, catalog.ErrFolderNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Library not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch library")
|
||||
return
|
||||
}
|
||||
|
||||
seriesCancelled := 0
|
||||
if h.SeriesMatchQueueRepo != nil {
|
||||
seriesCancelled, err = h.SeriesMatchQueueRepo.DeleteByFolder(r.Context(), id)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to cancel series queue", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel metadata matcher")
|
||||
return
|
||||
}
|
||||
}
|
||||
movieCancelled := 0
|
||||
if h.MovieMatchQueueRepo != nil {
|
||||
movieCancelled, err = h.MovieMatchQueueRepo.DeleteByFolder(r.Context(), id)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to cancel movie queue", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel metadata matcher")
|
||||
return
|
||||
}
|
||||
}
|
||||
rawFileCancelled := 0
|
||||
if h.RawMatchBacklogRepo != nil {
|
||||
rawFileCancelled, err = h.RawMatchBacklogRepo.SuppressUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode())
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to suppress raw backlog", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel metadata matcher")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
status, err := h.metadataMatchQueueStatus(r.Context(), id)
|
||||
if err != nil {
|
||||
slog.Error("metadata queue: failed to load cancelled queue status", "library_id", id, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, libraryMetadataMatchQueueActionResponse{
|
||||
Status: "cancelled",
|
||||
LibraryID: id,
|
||||
MovieCancelled: movieCancelled,
|
||||
SeriesCancelled: seriesCancelled,
|
||||
RawFileCancelled: rawFileCancelled,
|
||||
TotalCancelled: movieCancelled + seriesCancelled + rawFileCancelled,
|
||||
Queue: status,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) metadataMatchBacklogConfigured() bool {
|
||||
return h.folderRepo != nil &&
|
||||
(h.MovieMatchQueueRepo != nil || h.SeriesMatchQueueRepo != nil || h.RawMatchBacklogRepo != nil)
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) metadataMatchQueueStatus(ctx context.Context, libraryID int) (libraryMetadataMatchQueueStatusResponse, error) {
|
||||
resp := libraryMetadataMatchQueueStatusResponse{LibraryID: libraryID}
|
||||
if h.MovieMatchQueueRepo != nil {
|
||||
count, err := h.MovieMatchQueueRepo.CountByFolder(ctx, libraryID)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
resp.MovieCount = count
|
||||
}
|
||||
if h.SeriesMatchQueueRepo != nil {
|
||||
count, err := h.SeriesMatchQueueRepo.CountByFolder(ctx, libraryID)
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
resp.SeriesCount = count
|
||||
}
|
||||
if h.RawMatchBacklogRepo != nil {
|
||||
count, err := h.RawMatchBacklogRepo.CountUnmatchedMatchBacklogByFolder(ctx, libraryID, h.rawMatchBacklogMode())
|
||||
if err != nil {
|
||||
return resp, err
|
||||
}
|
||||
resp.RawFileCount = count
|
||||
}
|
||||
resp.TotalCount = resp.MovieCount + resp.SeriesCount + resp.RawFileCount
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (h *LibraryHandler) rawMatchBacklogMode() scanner.RawMatchBacklogMode {
|
||||
if h.TVSeriesRootQueue && h.MovieMatchQueueRepo != nil {
|
||||
return scanner.RawMatchBacklogMixed
|
||||
}
|
||||
if h.TVSeriesRootQueue {
|
||||
return scanner.RawMatchBacklogNonSeries
|
||||
}
|
||||
return scanner.RawMatchBacklogGeneric
|
||||
}
|
||||
|
||||
// HandleRefreshLibraryMetadata handles POST /libraries/{id}/refresh-metadata.
|
||||
// It queues a background admin job to refresh metadata for items in the
|
||||
// specified library. Quick mode is the default.
|
||||
@@ -2058,16 +2451,45 @@ func (h *LibraryHandler) HandleListUnmatchedItems(w http.ResponseWriter, r *http
|
||||
}
|
||||
}
|
||||
|
||||
// Optional case-insensitive search across title, library name, type, and
|
||||
// status. Applied server-side so it spans the whole table, not just the
|
||||
// current page. The displayed folder still comes from the lateral join below,
|
||||
// but the search predicate checks every membership so multi-library items are
|
||||
// found when any linked library name matches.
|
||||
search := strings.TrimSpace(q.Get("q"))
|
||||
filter := ""
|
||||
filterArgs := []any{}
|
||||
if search != "" {
|
||||
filterArgs = append(filterArgs, "%"+search+"%")
|
||||
filter = ` AND (
|
||||
mi.title ILIKE $1
|
||||
OR mi.type ILIKE $1
|
||||
OR mi.status ILIKE $1
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM media_item_libraries search_mil
|
||||
JOIN media_folders search_f ON search_f.id = search_mil.media_folder_id
|
||||
WHERE search_mil.content_id = mi.content_id
|
||||
AND search_f.name ILIKE $1
|
||||
)
|
||||
)`
|
||||
}
|
||||
|
||||
countSQL := `
|
||||
SELECT COUNT(*)
|
||||
FROM media_items mi
|
||||
WHERE mi.status IN ('unmatched', 'pending', 'ambiguous')`
|
||||
countSQL += filter
|
||||
|
||||
var total int
|
||||
if err := h.pool.QueryRow(r.Context(),
|
||||
`SELECT COUNT(*) FROM media_items WHERE status IN ('unmatched', 'pending', 'ambiguous')`,
|
||||
).Scan(&total); err != nil {
|
||||
if err := h.pool.QueryRow(r.Context(), countSQL, filterArgs...).Scan(&total); err != nil {
|
||||
slog.Error("counting unmatched items", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to count unmatched items")
|
||||
return
|
||||
}
|
||||
|
||||
rows, err := h.pool.Query(r.Context(), `
|
||||
listArgs := append(append([]any{}, filterArgs...), limit, offset)
|
||||
listSQL := fmt.Sprintf(`
|
||||
SELECT mi.content_id, mi.title, mi.year, mi.type, mi.status,
|
||||
COALESCE(lib.folder_id, 0),
|
||||
COALESCE(lib.folder_name, '')
|
||||
@@ -2079,10 +2501,12 @@ func (h *LibraryHandler) HandleListUnmatchedItems(w http.ResponseWriter, r *http
|
||||
WHERE mil.content_id = mi.content_id
|
||||
LIMIT 1
|
||||
) lib ON true
|
||||
WHERE mi.status IN ('unmatched', 'pending', 'ambiguous')
|
||||
WHERE mi.status IN ('unmatched', 'pending', 'ambiguous')%s
|
||||
ORDER BY mi.title ASC, mi.content_id ASC
|
||||
LIMIT $1 OFFSET $2
|
||||
`, limit, offset)
|
||||
LIMIT $%d OFFSET $%d
|
||||
`, filter, len(filterArgs)+1, len(filterArgs)+2)
|
||||
|
||||
rows, err := h.pool.Query(r.Context(), listSQL, listArgs...)
|
||||
if err != nil {
|
||||
slog.Error("listing unmatched items", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list unmatched items")
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// TestLibraryMatchUnmatchedItems_NilPool verifies that the unmatched-items
|
||||
@@ -166,6 +169,79 @@ func TestLibraryMatchRematch_DeprecatedStillRoutes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryMetadataMatchQueueHandlers_NilFolderRepo(t *testing.T) {
|
||||
h := &LibraryHandler{MovieMatchQueueRepo: noopMovieMatchQueue{}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
path string
|
||||
handler http.HandlerFunc
|
||||
}{
|
||||
{
|
||||
name: "get",
|
||||
method: http.MethodGet,
|
||||
path: "/libraries/1/metadata-match-queue",
|
||||
handler: h.HandleGetMetadataMatchQueue,
|
||||
},
|
||||
{
|
||||
name: "retry",
|
||||
method: http.MethodPost,
|
||||
path: "/libraries/1/metadata-match-queue/retry",
|
||||
handler: h.HandleRetryMetadataMatchQueue,
|
||||
},
|
||||
{
|
||||
name: "cancel",
|
||||
method: http.MethodDelete,
|
||||
path: "/libraries/1/metadata-match-queue",
|
||||
handler: h.HandleCancelMetadataMatchQueue,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := chi.NewRouter()
|
||||
r.Method(tt.method, "/libraries/{id}/metadata-match-queue", tt.handler)
|
||||
r.Method(tt.method, "/libraries/{id}/metadata-match-queue/retry", tt.handler)
|
||||
|
||||
req := httptest.NewRequest(tt.method, tt.path, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
r.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expected 503 for nil folderRepo, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp errorResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decoding error response: %v", err)
|
||||
}
|
||||
if resp.Error != "unavailable" {
|
||||
t.Errorf("expected error code 'unavailable', got %q", resp.Error)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type noopMovieMatchQueue struct{}
|
||||
|
||||
func (noopMovieMatchQueue) SyncForFolder(context.Context, int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (noopMovieMatchQueue) DeleteByFolder(context.Context, int) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (noopMovieMatchQueue) CountByFolder(context.Context, int) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (noopMovieMatchQueue) ListByFolder(context.Context, int, int, int) ([]models.MovieMatchQueueEntry, int, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
func keys(m map[string]any) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
|
||||
@@ -39,6 +39,8 @@ var personRefreshRate = ratelimit.Rate{
|
||||
Burst: 10,
|
||||
}
|
||||
|
||||
const personMetadataStaleAfter = 90 * 24 * time.Hour
|
||||
|
||||
// PeopleHandler serves person-related API endpoints.
|
||||
type PeopleHandler struct {
|
||||
personRepo peopleRepository
|
||||
@@ -131,6 +133,8 @@ func (h *PeopleHandler) HandleGetPerson(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
h.enqueuePersonRefreshIfDue(*person)
|
||||
|
||||
writeJSON(w, http.StatusOK, h.toResponse(r.Context(), *person))
|
||||
}
|
||||
|
||||
@@ -359,6 +363,28 @@ func (h *PeopleHandler) toResponse(ctx context.Context, p models.Person) personR
|
||||
return resp
|
||||
}
|
||||
|
||||
func (h *PeopleHandler) enqueuePersonRefreshIfDue(person models.Person) {
|
||||
if h.refreshQueue == nil || !personHasRefreshableProviderID(person) {
|
||||
return
|
||||
}
|
||||
|
||||
if personMetadataIncomplete(person) {
|
||||
return
|
||||
}
|
||||
|
||||
if person.UpdatedAt.Before(time.Now().Add(-personMetadataStaleAfter)) {
|
||||
h.refreshQueue.Enqueue(person.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func personHasRefreshableProviderID(person models.Person) bool {
|
||||
return person.TmdbID != "" || person.ImdbID != "" || person.TvdbID != ""
|
||||
}
|
||||
|
||||
func personMetadataIncomplete(person models.Person) bool {
|
||||
return person.Bio == "" || person.PhotoPath == "" || person.PhotoPath == "-" || person.BirthDate == nil
|
||||
}
|
||||
|
||||
func parseOptionalPersonDate(raw string) (*time.Time, error) {
|
||||
if raw == "" {
|
||||
return nil, nil
|
||||
|
||||
@@ -305,6 +305,7 @@ type subtitleURL struct {
|
||||
Forced bool `json:"forced"`
|
||||
HearingImpaired bool `json:"hearing_impaired"`
|
||||
URL string `json:"url"`
|
||||
FontBundleURL string `json:"font_bundle_url,omitempty"`
|
||||
}
|
||||
|
||||
// changeAudioRequest represents the JSON body for PATCH /playback/{session_id}/audio.
|
||||
@@ -1403,7 +1404,11 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req
|
||||
if resp.SubtitleURLs[i].Source == "embedded" {
|
||||
// Pass the ffmpeg-relative subtitle stream index to the proxy.
|
||||
embeddedIdx := resp.SubtitleURLs[i].Index - embeddedOffset
|
||||
resp.SubtitleURLs[i].URL = proxyNode.URL + "/stream/subtitles/" + token + "/" + strconv.Itoa(embeddedIdx) + subtitleURLExt(resp.SubtitleURLs[i].Codec)
|
||||
proxySubtitleURL := proxyNode.URL + "/stream/subtitles/" + token + "/" + strconv.Itoa(embeddedIdx)
|
||||
resp.SubtitleURLs[i].URL = proxySubtitleURL + subtitleURLExt(resp.SubtitleURLs[i].Codec)
|
||||
if resp.SubtitleURLs[i].FontBundleURL != "" {
|
||||
resp.SubtitleURLs[i].FontBundleURL = proxySubtitleURL + "/fonts"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1459,6 +1464,7 @@ func buildSubtitleURLs(sessionID string, file *models.MediaFile, downloaded []su
|
||||
Forced: track.Forced,
|
||||
HearingImpaired: track.HearingImpaired,
|
||||
URL: fmt.Sprintf("/stream/%s/subtitles/%d%s", sessionID, embeddedOffset+i, subtitleURLExt(track.Codec)),
|
||||
FontBundleURL: subtitleFontBundleURL(sessionID, embeddedOffset+i, track.Codec),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1478,6 +1484,13 @@ func buildSubtitleURLs(sessionID string, file *models.MediaFile, downloaded []su
|
||||
return urls
|
||||
}
|
||||
|
||||
func subtitleFontBundleURL(sessionID string, trackIndex int, codec string) string {
|
||||
if !playback.IsASS(codec) {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("/stream/%s/subtitles/%d/fonts", sessionID, trackIndex)
|
||||
}
|
||||
|
||||
func firstNonEmptyString(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
@@ -1531,6 +1544,7 @@ func (h *PlaybackHandler) HandleUpdateProgress(w http.ResponseWriter, r *http.Re
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update progress")
|
||||
return
|
||||
}
|
||||
h.syncSessionsNow(r.Context(), "progress")
|
||||
|
||||
// Persist progress to UserStore (best-effort).
|
||||
if sess, getErr := h.sessionMgr.GetSession(sessionID); getErr == nil {
|
||||
@@ -1838,6 +1852,7 @@ type transcodeStartState struct {
|
||||
file *models.MediaFile
|
||||
session *playback.Session
|
||||
switchedFileID *int
|
||||
hwAccel string
|
||||
}
|
||||
|
||||
// finalizeTranscodeStart updates the playback session state after a transcode
|
||||
@@ -1878,6 +1893,7 @@ func (h *PlaybackHandler) finalizeTranscodeStart(r *http.Request, st transcodeSt
|
||||
TargetVideoCodec: st.req.TargetCodecVideo,
|
||||
TargetAudioCodec: st.req.TargetCodecAudio,
|
||||
TargetBitrateKbps: st.req.TargetBitrateKbps,
|
||||
TranscodeHWAccel: st.hwAccel,
|
||||
}); err != nil {
|
||||
slog.Error("failed to update transcode stream state", "session", st.req.SessionID, "error", err, "playback_session_id", st.req.SessionID)
|
||||
}
|
||||
@@ -2052,16 +2068,35 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re
|
||||
writeError(w, http.StatusBadGateway, "transcode_node_unavailable", "Transcode node is unavailable")
|
||||
return
|
||||
}
|
||||
resp.Body.Close()
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
slog.Error("remote transcode start rejected", "status", resp.StatusCode, "node", tcNode.URL)
|
||||
writeError(w, http.StatusBadGateway, "transcode_start_failed", "Transcode node rejected the request")
|
||||
return
|
||||
}
|
||||
var nodeResp transcodenode.TranscodeStartResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&nodeResp); err != nil {
|
||||
slog.Warn("remote transcode start response decode failed",
|
||||
"error", err,
|
||||
"node", tcNode.URL,
|
||||
"session", req.SessionID,
|
||||
"playback_session_id", req.SessionID,
|
||||
)
|
||||
}
|
||||
effectiveHWAccel := strings.TrimSpace(nodeResp.HWAccel)
|
||||
if effectiveHWAccel == "" {
|
||||
effectiveHWAccel = strings.TrimSpace(nodeReq.HWAccel)
|
||||
}
|
||||
|
||||
manifestURL := h.buildProxyManifestURL(req.SessionID, session, tcNode.URL)
|
||||
h.finalizeTranscodeStart(r, transcodeStartState{req: req, file: file, session: session, switchedFileID: switchedFileID})
|
||||
h.finalizeTranscodeStart(r, transcodeStartState{
|
||||
req: req,
|
||||
file: file,
|
||||
session: session,
|
||||
switchedFileID: switchedFileID,
|
||||
hwAccel: effectiveHWAccel,
|
||||
})
|
||||
writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL))
|
||||
return
|
||||
}
|
||||
@@ -2109,7 +2144,13 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re
|
||||
h.monitorLocalTranscodeExit(req.SessionID, transcodeSession)
|
||||
|
||||
manifestURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", req.SessionID)
|
||||
h.finalizeTranscodeStart(r, transcodeStartState{req: req, file: file, session: session, switchedFileID: switchedFileID})
|
||||
h.finalizeTranscodeStart(r, transcodeStartState{
|
||||
req: req,
|
||||
file: file,
|
||||
session: session,
|
||||
switchedFileID: switchedFileID,
|
||||
hwAccel: transcodeSession.Opts().HWAccel,
|
||||
})
|
||||
writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL))
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ type playbackSessionRow struct {
|
||||
FileDuration *int `json:"file_duration"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
PositionSeconds float64 `json:"position_seconds"`
|
||||
IsPaused bool `json:"is_paused"`
|
||||
HasPlaybackControl bool `json:"has_playback_control"`
|
||||
ClientIP string `json:"client_ip,omitempty"`
|
||||
@@ -53,6 +54,7 @@ type playbackSessionRow struct {
|
||||
TargetVideoCodec string `json:"target_video_codec,omitempty"`
|
||||
TargetAudioCodec string `json:"target_audio_codec,omitempty"`
|
||||
TargetBitrateKbps *int `json:"target_bitrate_kbps"`
|
||||
TranscodeHWAccel string `json:"transcode_hw_accel,omitempty"`
|
||||
SourceContainer string `json:"source_container,omitempty"`
|
||||
SourceBitrateKbps *int `json:"source_bitrate_kbps"`
|
||||
SourceVideoCodec string `json:"source_video_codec,omitempty"`
|
||||
@@ -143,6 +145,7 @@ func (l *PlaybackSessionsLoader) Load(
|
||||
mf.duration,
|
||||
s.started_at,
|
||||
s.updated_at,
|
||||
COALESCE(s.position_seconds, 0),
|
||||
COALESCE(s.is_paused, FALSE),
|
||||
COALESCE(s.has_websocket, FALSE),
|
||||
COALESCE(HOST(s.client_ip), ''),
|
||||
@@ -154,6 +157,7 @@ func (l *PlaybackSessionsLoader) Load(
|
||||
COALESCE(s.target_video_codec, ''),
|
||||
COALESCE(s.target_audio_codec, ''),
|
||||
s.target_bitrate_kbps,
|
||||
COALESCE(s.transcode_hw_accel, ''),
|
||||
COALESCE(mf.container, ''),
|
||||
mf.bitrate,
|
||||
COALESCE(mf.codec_video, ''),
|
||||
@@ -199,9 +203,9 @@ func (l *PlaybackSessionsLoader) Load(
|
||||
&s.MediaTitle, &s.MediaType, &s.SeriesName, &s.EpisodeName, &s.SeasonNumber, &s.EpisodeNumber,
|
||||
&posterPath,
|
||||
&s.PlayMethod, &s.ReportingNode, &s.NodeDisplayName, &s.FileDuration, &s.StartedAt, &s.UpdatedAt,
|
||||
&s.IsPaused, &s.HasPlaybackControl, &s.ClientIP, &s.AudioTrackIndex, &s.TranscodeAudio, &streamBitrateKbps,
|
||||
&s.PositionSeconds, &s.IsPaused, &s.HasPlaybackControl, &s.ClientIP, &s.AudioTrackIndex, &s.TranscodeAudio, &streamBitrateKbps,
|
||||
&s.TranscodeNodeURL, &s.TargetResolution, &s.TargetVideoCodec, &s.TargetAudioCodec, &targetBitrateKbps,
|
||||
&s.SourceContainer, &sourceBitrateKbps, &s.SourceVideoCodec, &s.SourceVideoResolution,
|
||||
&s.TranscodeHWAccel, &s.SourceContainer, &sourceBitrateKbps, &s.SourceVideoCodec, &s.SourceVideoResolution,
|
||||
&s.SourceAudioCodec, &sourceAudioChannels, &audioTracksJSON, &s.RequestedVideoCodec, &s.RequestedVideoResolution,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scanning playback session: %w", err)
|
||||
@@ -314,7 +318,7 @@ func sessionComponentDecision(playMethod string, transcodeAudio bool, targetVide
|
||||
}
|
||||
audioDec := "transcode"
|
||||
if !transcodeAudio {
|
||||
audioDec = "direct"
|
||||
audioDec = "remux"
|
||||
}
|
||||
return videoDec, audioDec
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package handlers
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T) {
|
||||
videoDecision, audioDecision := sessionComponentDecision("transcode", false, "copy")
|
||||
|
||||
if videoDecision != "remux" {
|
||||
t.Fatalf("videoDecision = %q, want remux", videoDecision)
|
||||
}
|
||||
if audioDecision != "remux" {
|
||||
t.Fatalf("audioDecision = %q, want remux", audioDecision)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -20,9 +21,14 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
"github.com/Silo-Server/silo-server/internal/pluginhost"
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
"github.com/Silo-Server/silo-server/internal/uploads"
|
||||
)
|
||||
|
||||
const maxPluginUploadSize = 256 << 20
|
||||
const (
|
||||
maxPluginUploadSize = 256 << 20
|
||||
maxPluginUploadChunkSize = 1 << 20
|
||||
defaultPluginChunkSize = 512 << 10
|
||||
)
|
||||
|
||||
type PluginHandler struct {
|
||||
repositories *plugins.RepositoryStore
|
||||
@@ -33,6 +39,7 @@ type PluginHandler struct {
|
||||
proxy *plugins.HTTPProxy
|
||||
chainRepo *metadata.ChainRepository
|
||||
imageResolver *metadata.PluginImageResolver
|
||||
uploads *uploads.Manager
|
||||
}
|
||||
|
||||
func NewPluginHandler(
|
||||
@@ -54,6 +61,10 @@ func NewPluginHandler(
|
||||
proxy: proxy,
|
||||
chainRepo: chainRepo,
|
||||
imageResolver: imageResolver,
|
||||
uploads: uploads.NewManager(uploads.ManagerOptions{
|
||||
MaxSize: maxPluginUploadSize,
|
||||
MaxChunkSize: maxPluginUploadChunkSize,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +108,12 @@ type userPluginSettingsRequest struct {
|
||||
Values map[string]string `json:"values"`
|
||||
}
|
||||
|
||||
type pluginChunkedUploadCreateRequest struct {
|
||||
Filename string `json:"filename"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ChunkSize int64 `json:"chunk_size,omitempty"`
|
||||
}
|
||||
|
||||
type pluginRepositoryResponse struct {
|
||||
ID int `json:"id"`
|
||||
URL string `json:"url"`
|
||||
@@ -248,6 +265,18 @@ type pluginTaskBindingUpdateResponse struct {
|
||||
RestartRequired bool `json:"restart_required"`
|
||||
}
|
||||
|
||||
type pluginChunkedUploadSessionResponse struct {
|
||||
UploadID string `json:"upload_id"`
|
||||
Filename string `json:"filename"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ChunkSize int64 `json:"chunk_size"`
|
||||
TotalChunks int `json:"total_chunks"`
|
||||
ReceivedChunks int `json:"received_chunks"`
|
||||
ReceivedBytes int64 `json:"received_bytes"`
|
||||
Complete bool `json:"complete"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleListRepositories(w http.ResponseWriter, r *http.Request) {
|
||||
repositories, err := h.repositories.List(r.Context())
|
||||
if err != nil {
|
||||
@@ -491,27 +520,114 @@ func (h *PluginHandler) HandleUploadInstallation(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
uploadData, err := os.ReadFile(tempPath)
|
||||
if err != nil {
|
||||
slog.Error("reading temp plugin upload file", "filename", header.Filename, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to process plugin upload")
|
||||
return
|
||||
}
|
||||
|
||||
var result *plugins.InstallResult
|
||||
if isZipUpload(uploadData) {
|
||||
result, err = h.service.InstallLocal(r.Context(), plugins.InstallArchiveRequest{
|
||||
ArchivePath: tempPath,
|
||||
})
|
||||
} else {
|
||||
result, err = h.service.InstallBinaryUpload(r.Context(), uploadData)
|
||||
}
|
||||
result, err := h.installUploadedPlugin(r.Context(), tempPath)
|
||||
if err != nil {
|
||||
slog.Error("installing uploaded plugin", "filename", header.Filename, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to install uploaded plugin")
|
||||
return
|
||||
}
|
||||
|
||||
h.writeUploadedPluginResponse(w, r, result)
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleCreateChunkedUpload(w http.ResponseWriter, r *http.Request) {
|
||||
var req pluginChunkedUploadCreateRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.ChunkSize == 0 {
|
||||
req.ChunkSize = defaultPluginChunkSize
|
||||
}
|
||||
|
||||
session, err := h.uploads.Create(uploads.CreateRequest{
|
||||
Filename: req.Filename,
|
||||
SizeBytes: req.SizeBytes,
|
||||
ChunkSize: req.ChunkSize,
|
||||
})
|
||||
if err != nil {
|
||||
status, message := uploadErrorResponse(err)
|
||||
writeError(w, status, "upload_error", message)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, toPluginChunkedUploadSessionResponse(session))
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleUploadChunk(w http.ResponseWriter, r *http.Request) {
|
||||
uploadID := chi.URLParam(r, "upload_id")
|
||||
chunkIndex, err := strconv.Atoi(chi.URLParam(r, "chunk_index"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid chunk index")
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, h.uploads.MaxChunkSize()+1)
|
||||
defer r.Body.Close()
|
||||
|
||||
session, err := h.uploads.PutChunk(r.Context(), uploadID, chunkIndex, r.Body, r.ContentLength)
|
||||
if err != nil {
|
||||
status, message := uploadErrorResponse(err)
|
||||
writeError(w, status, "upload_error", message)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, toPluginChunkedUploadSessionResponse(session))
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleCompleteChunkedUpload(w http.ResponseWriter, r *http.Request) {
|
||||
uploadID := chi.URLParam(r, "upload_id")
|
||||
upload, err := h.uploads.Complete(uploadID)
|
||||
if err != nil {
|
||||
status, message := uploadErrorResponse(err)
|
||||
writeError(w, status, "upload_error", message)
|
||||
return
|
||||
}
|
||||
defer upload.Cleanup()
|
||||
|
||||
result, err := h.installUploadedPlugin(r.Context(), upload.Path)
|
||||
if err != nil {
|
||||
slog.Error("installing chunked plugin upload",
|
||||
"filename", upload.Filename,
|
||||
"upload_id", upload.ID,
|
||||
"error", err,
|
||||
)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to install uploaded plugin")
|
||||
return
|
||||
}
|
||||
|
||||
h.writeUploadedPluginResponse(w, r, result)
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleCancelChunkedUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if err := h.uploads.Cancel(chi.URLParam(r, "upload_id")); err != nil && !errors.Is(err, uploads.ErrNotFound) {
|
||||
status, message := uploadErrorResponse(err)
|
||||
writeError(w, status, "upload_error", message)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *PluginHandler) installUploadedPlugin(ctx context.Context, path string) (*plugins.InstallResult, error) {
|
||||
zipUpload, err := isZipUploadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if zipUpload {
|
||||
return h.service.InstallLocal(ctx, plugins.InstallArchiveRequest{
|
||||
ArchivePath: path,
|
||||
})
|
||||
}
|
||||
|
||||
uploadData, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read uploaded plugin binary: %w", err)
|
||||
}
|
||||
return h.service.InstallBinaryUpload(ctx, uploadData)
|
||||
}
|
||||
|
||||
func (h *PluginHandler) writeUploadedPluginResponse(w http.ResponseWriter, r *http.Request, result *plugins.InstallResult) {
|
||||
h.syncMetadataProviders(r.Context(), result.Installation)
|
||||
h.syncImageResolvers(r.Context(), result.Installation)
|
||||
|
||||
@@ -593,6 +709,55 @@ func isZipUpload(data []byte) bool {
|
||||
bytes.Equal(data[:4], []byte("PK\x07\x08"))
|
||||
}
|
||||
|
||||
func isZipUploadFile(path string) (bool, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("open uploaded plugin file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var header [4]byte
|
||||
n, err := io.ReadFull(file, header[:])
|
||||
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return false, fmt.Errorf("read uploaded plugin header: %w", err)
|
||||
}
|
||||
return isZipUpload(header[:n]), nil
|
||||
}
|
||||
|
||||
func toPluginChunkedUploadSessionResponse(session uploads.SessionInfo) pluginChunkedUploadSessionResponse {
|
||||
return pluginChunkedUploadSessionResponse{
|
||||
UploadID: session.ID,
|
||||
Filename: session.Filename,
|
||||
SizeBytes: session.SizeBytes,
|
||||
ChunkSize: session.ChunkSize,
|
||||
TotalChunks: session.TotalChunks,
|
||||
ReceivedChunks: session.ReceivedChunks,
|
||||
ReceivedBytes: session.ReceivedBytes,
|
||||
Complete: session.Complete,
|
||||
ExpiresAt: session.ExpiresAt,
|
||||
}
|
||||
}
|
||||
|
||||
func uploadErrorResponse(err error) (int, string) {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
switch {
|
||||
case errors.As(err, &maxBytesErr), errors.Is(err, uploads.ErrTooLarge):
|
||||
return http.StatusRequestEntityTooLarge, "Upload exceeds the maximum allowed size"
|
||||
case errors.Is(err, uploads.ErrNotFound):
|
||||
return http.StatusNotFound, "Upload session not found"
|
||||
case errors.Is(err, uploads.ErrExpired):
|
||||
return http.StatusGone, "Upload session expired"
|
||||
case errors.Is(err, uploads.ErrIncomplete):
|
||||
return http.StatusConflict, "Upload session is incomplete"
|
||||
case errors.Is(err, uploads.ErrAlreadyCompleted):
|
||||
return http.StatusConflict, "Upload session is already complete"
|
||||
case errors.Is(err, uploads.ErrInvalidChunk), errors.Is(err, uploads.ErrInvalidRequest):
|
||||
return http.StatusBadRequest, err.Error()
|
||||
default:
|
||||
return http.StatusInternalServerError, "Failed to process upload"
|
||||
}
|
||||
}
|
||||
|
||||
func (h *PluginHandler) HandleUpdateInstallation(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := parseNamedIDParam(r, "id")
|
||||
if err != nil {
|
||||
|
||||
@@ -258,7 +258,7 @@ func (h *ProgressHandler) HandleSyncProgress(w http.ResponseWriter, r *http.Requ
|
||||
item.MediaItemID,
|
||||
"",
|
||||
"progress",
|
||||
"progress.updated",
|
||||
userStateEventState{},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -789,12 +789,9 @@ func (h *RecommendationsHandler) blendUpcomingIntoDiscoverRows(
|
||||
events, err := h.CalendarRepo.ListEvents(ctx, catalog.CalendarFilter{
|
||||
Start: start,
|
||||
End: end,
|
||||
Filter: "all",
|
||||
AllowedLibraryIDs: filter.AllowedLibraryIDs,
|
||||
DisabledLibraryIDs: filter.DisabledLibraryIDs,
|
||||
MaxContentRating: filter.MaxContentRating,
|
||||
UserID: userID,
|
||||
ProfileID: profileID,
|
||||
})
|
||||
if err != nil {
|
||||
return discoverRows, err
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// forwardedHost returns the public host a reverse proxy or CDN forwarded the
|
||||
// request on behalf of, taken from the first X-Forwarded-Host value, or "" when
|
||||
// the header is absent.
|
||||
//
|
||||
// Behind a TLS-terminating CDN/proxy, r.Host is the internal origin host the
|
||||
// proxy dialed, while the public host the browser actually used arrives here.
|
||||
// Callers that need to reason about the client-facing host (origin checks,
|
||||
// absolute URL construction) must prefer this over r.Host.
|
||||
func forwardedHost(r *http.Request) string {
|
||||
forwarded := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
|
||||
if forwarded == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Multiple proxy hops produce a comma-separated list; the first entry is
|
||||
// the original client-facing host.
|
||||
if comma := strings.IndexByte(forwarded, ','); comma >= 0 {
|
||||
forwarded = forwarded[:comma]
|
||||
}
|
||||
|
||||
return strings.TrimSpace(forwarded)
|
||||
}
|
||||
@@ -35,8 +35,9 @@ type RequestService interface {
|
||||
GetUserLimit(ctx context.Context, viewer mediarequests.Viewer, userID int) (*mediarequests.UserLimit, error)
|
||||
UpsertUserLimit(ctx context.Context, viewer mediarequests.Viewer, limit mediarequests.UserLimit) (*mediarequests.UserLimit, error)
|
||||
ListIntegrations(ctx context.Context, viewer mediarequests.Viewer) ([]mediarequests.Integration, error)
|
||||
UpsertIntegration(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error)
|
||||
UpsertIntegrations(ctx context.Context, viewer mediarequests.Viewer, integrations []mediarequests.Integration) ([]mediarequests.Integration, error)
|
||||
CreateIntegration(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error)
|
||||
UpdateIntegration(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error)
|
||||
DeleteIntegration(ctx context.Context, viewer mediarequests.Viewer, id string) error
|
||||
LoadIntegrationOptions(ctx context.Context, viewer mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.IntegrationOptions, error)
|
||||
|
||||
ListStudios(ctx context.Context, viewer mediarequests.Viewer) ([]mediarequests.DiscoverBrandCard, error)
|
||||
@@ -422,26 +423,53 @@ func (h *RequestsHandler) HandleListIntegrations(w http.ResponseWriter, r *http.
|
||||
}{Integrations: toIntegrationResponses(integrations)})
|
||||
}
|
||||
|
||||
func (h *RequestsHandler) HandleUpdateIntegrations(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *RequestsHandler) HandleCreateIntegration(w http.ResponseWriter, r *http.Request) {
|
||||
viewer, ok := requestViewer(w, r, false)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Integrations []mediarequests.Integration `json:"integrations"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
var integration mediarequests.Integration
|
||||
if err := json.NewDecoder(r.Body).Decode(&integration); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
updated, err := h.service.UpsertIntegrations(r.Context(), viewer, body.Integrations)
|
||||
created, err := h.service.CreateIntegration(r.Context(), viewer, integration)
|
||||
if err != nil {
|
||||
writeRequestServiceError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, struct {
|
||||
Integrations []requestIntegrationResponse `json:"integrations"`
|
||||
}{Integrations: toIntegrationResponses(updated)})
|
||||
writeJSON(w, http.StatusCreated, requestIntegrationResponseFrom(*created))
|
||||
}
|
||||
|
||||
func (h *RequestsHandler) HandleUpdateIntegration(w http.ResponseWriter, r *http.Request) {
|
||||
viewer, ok := requestViewer(w, r, false)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var integration mediarequests.Integration
|
||||
if err := json.NewDecoder(r.Body).Decode(&integration); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
integration.ID = chi.URLParam(r, "id")
|
||||
updated, err := h.service.UpdateIntegration(r.Context(), viewer, integration)
|
||||
if err != nil {
|
||||
writeRequestServiceError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, requestIntegrationResponseFrom(*updated))
|
||||
}
|
||||
|
||||
func (h *RequestsHandler) HandleDeleteIntegration(w http.ResponseWriter, r *http.Request) {
|
||||
viewer, ok := requestViewer(w, r, false)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := h.service.DeleteIntegration(r.Context(), viewer, chi.URLParam(r, "id")); err != nil {
|
||||
writeRequestServiceError(w, err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *RequestsHandler) HandleLoadIntegrationOptions(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -456,7 +484,9 @@ func (h *RequestsHandler) HandleLoadIntegrationOptions(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
}
|
||||
integration.Kind = chi.URLParam(r, "kind")
|
||||
if id := strings.TrimSpace(chi.URLParam(r, "id")); id != "" {
|
||||
integration.ID = id
|
||||
}
|
||||
options, err := h.service.LoadIntegrationOptions(r.Context(), viewer, integration)
|
||||
if err != nil {
|
||||
writeRequestServiceError(w, err)
|
||||
@@ -558,37 +588,69 @@ func parsePositivePathInt(w http.ResponseWriter, r *http.Request, key string) (i
|
||||
}
|
||||
|
||||
type requestIntegrationResponse struct {
|
||||
Kind string `json:"kind"`
|
||||
Enabled bool `json:"enabled"`
|
||||
BaseURL string `json:"base_url"`
|
||||
HasAPIKey bool `json:"has_api_key"`
|
||||
RootFolder string `json:"root_folder"`
|
||||
QualityProfileID *int `json:"quality_profile_id,omitempty"`
|
||||
Tags []int `json:"tags"`
|
||||
Options map[string]any `json:"options"`
|
||||
LastCheckAt *time.Time `json:"last_check_at,omitempty"`
|
||||
LastCheckStatus string `json:"last_check_status,omitempty"`
|
||||
LastCheckError string `json:"last_check_error,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Is4K bool `json:"is_4k"`
|
||||
IsDefault bool `json:"is_default"`
|
||||
IsDefault4K bool `json:"is_default_4k"`
|
||||
AnimeEnabled bool `json:"anime_enabled"`
|
||||
AnimeQualityProfileID *int `json:"anime_quality_profile_id,omitempty"`
|
||||
AnimeRootFolder string `json:"anime_root_folder,omitempty"`
|
||||
AnimeTags []int `json:"anime_tags"`
|
||||
BaseURL string `json:"base_url"`
|
||||
HasAPIKey bool `json:"has_api_key"`
|
||||
RootFolder string `json:"root_folder"`
|
||||
QualityProfileID *int `json:"quality_profile_id,omitempty"`
|
||||
Tags []int `json:"tags"`
|
||||
Options map[string]any `json:"options"`
|
||||
LastCheckAt *time.Time `json:"last_check_at,omitempty"`
|
||||
LastCheckStatus string `json:"last_check_status,omitempty"`
|
||||
LastCheckError string `json:"last_check_error,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func requestIntegrationResponseFrom(integration mediarequests.Integration) requestIntegrationResponse {
|
||||
// Normalize nil slices so they serialize as [] (not null); the frontend
|
||||
// types them as number[] and indexes/maps over them.
|
||||
tags := integration.Tags
|
||||
if tags == nil {
|
||||
tags = []int{}
|
||||
}
|
||||
animeTags := integration.AnimeTags
|
||||
if animeTags == nil {
|
||||
animeTags = []int{}
|
||||
}
|
||||
return requestIntegrationResponse{
|
||||
ID: integration.ID,
|
||||
Name: integration.Name,
|
||||
Kind: integration.Kind,
|
||||
Enabled: integration.Enabled,
|
||||
Is4K: integration.Is4K,
|
||||
IsDefault: integration.IsDefault,
|
||||
IsDefault4K: integration.IsDefault4K,
|
||||
AnimeEnabled: integration.AnimeEnabled,
|
||||
AnimeQualityProfileID: integration.AnimeQualityProfileID,
|
||||
AnimeRootFolder: integration.AnimeRootFolder,
|
||||
AnimeTags: animeTags,
|
||||
BaseURL: integration.BaseURL,
|
||||
HasAPIKey: strings.TrimSpace(integration.APIKeyRef) != "",
|
||||
RootFolder: integration.RootFolder,
|
||||
QualityProfileID: integration.QualityProfileID,
|
||||
Tags: tags,
|
||||
Options: integration.Options,
|
||||
LastCheckAt: integration.LastCheckAt,
|
||||
LastCheckStatus: integration.LastCheckStatus,
|
||||
LastCheckError: integration.LastCheckError,
|
||||
UpdatedAt: integration.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func toIntegrationResponses(integrations []mediarequests.Integration) []requestIntegrationResponse {
|
||||
out := make([]requestIntegrationResponse, 0, len(integrations))
|
||||
for _, integration := range integrations {
|
||||
out = append(out, requestIntegrationResponse{
|
||||
Kind: integration.Kind,
|
||||
Enabled: integration.Enabled,
|
||||
BaseURL: integration.BaseURL,
|
||||
HasAPIKey: strings.TrimSpace(integration.APIKeyRef) != "",
|
||||
RootFolder: integration.RootFolder,
|
||||
QualityProfileID: integration.QualityProfileID,
|
||||
Tags: integration.Tags,
|
||||
Options: integration.Options,
|
||||
LastCheckAt: integration.LastCheckAt,
|
||||
LastCheckStatus: integration.LastCheckStatus,
|
||||
LastCheckError: integration.LastCheckError,
|
||||
UpdatedAt: integration.UpdatedAt,
|
||||
})
|
||||
out = append(out, requestIntegrationResponseFrom(integration))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -127,12 +127,16 @@ func (f *fakeRequestService) ListIntegrations(context.Context, mediarequests.Vie
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeRequestService) UpsertIntegration(context.Context, mediarequests.Viewer, mediarequests.Integration) (*mediarequests.Integration, error) {
|
||||
return nil, nil
|
||||
func (f *fakeRequestService) CreateIntegration(_ context.Context, _ mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error) {
|
||||
return &integration, nil
|
||||
}
|
||||
|
||||
func (f *fakeRequestService) UpsertIntegrations(context.Context, mediarequests.Viewer, []mediarequests.Integration) ([]mediarequests.Integration, error) {
|
||||
return nil, nil
|
||||
func (f *fakeRequestService) UpdateIntegration(_ context.Context, _ mediarequests.Viewer, integration mediarequests.Integration) (*mediarequests.Integration, error) {
|
||||
return &integration, nil
|
||||
}
|
||||
|
||||
func (f *fakeRequestService) DeleteIntegration(context.Context, mediarequests.Viewer, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRequestService) LoadIntegrationOptions(context.Context, mediarequests.Viewer, mediarequests.Integration) (*mediarequests.IntegrationOptions, error) {
|
||||
|
||||
@@ -1359,15 +1359,7 @@ func (h *SectionHandler) toSectionItemResponse(sectionType sections.SectionType,
|
||||
// uses the cached w1280 backdrop for Continue Watching / Next Up rows.
|
||||
func sectionBackdropPath(sectionType sections.SectionType, path string) string {
|
||||
if sectionType == sections.SectionContinueWatching || sectionType == sections.SectionNextUp {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return path
|
||||
}
|
||||
// Plugin-prefixed paths pass through; the resolver still receives the
|
||||
// existing "featured" semantic variant.
|
||||
if strings.Contains(path, "://") {
|
||||
return path
|
||||
}
|
||||
return strings.Replace(path, "/original.", "/w1280.", 1)
|
||||
return catalog.BackdropVariantPath(path, "w1280")
|
||||
}
|
||||
return featuredBackdropPath(path)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,18 @@ func TestSectionBackdropPathUsesExpectedVariants(t *testing.T) {
|
||||
path: "/tmdb/shows/1399/backdrop/original.jpg",
|
||||
want: "/tmdb/shows/1399/backdrop/w1920.jpg",
|
||||
},
|
||||
{
|
||||
name: "continue watching episode still clamps to w500",
|
||||
sectionType: sections.SectionContinueWatching,
|
||||
path: "tvdb/series/73141/seasons/22/episodes/9/still/original.webp",
|
||||
want: "tvdb/series/73141/seasons/22/episodes/9/still/w500.webp",
|
||||
},
|
||||
{
|
||||
name: "featured section episode still clamps to w500",
|
||||
sectionType: sections.SectionRecentlyAdded,
|
||||
path: "tvdb/series/73141/seasons/22/episodes/9/still/original.webp",
|
||||
want: "tvdb/series/73141/seasons/22/episodes/9/still/w500.webp",
|
||||
},
|
||||
{
|
||||
name: "http paths pass through",
|
||||
sectionType: sections.SectionContinueWatching,
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -16,6 +17,10 @@ import (
|
||||
)
|
||||
|
||||
const subtitleAppearanceSettingKey = "subtitle_appearance"
|
||||
const (
|
||||
libraryPageStateSettingKey = "ui.library_page_state"
|
||||
rememberLibraryPageStateSettingKey = "ui.remember_library_page_state"
|
||||
)
|
||||
|
||||
const (
|
||||
deviceIDHeader = "X-Silo-Device-Id"
|
||||
@@ -156,6 +161,16 @@ var settingsRegistry = map[string]settingSpec{
|
||||
DefaultValue: "",
|
||||
Validate: validateJSONSetting(subtitleAppearanceSettingKey),
|
||||
},
|
||||
libraryPageStateSettingKey: {
|
||||
Scope: scopeDevice,
|
||||
DefaultValue: "",
|
||||
Validate: validateJSONSetting(libraryPageStateSettingKey),
|
||||
},
|
||||
rememberLibraryPageStateSettingKey: {
|
||||
Scope: scopeDevice,
|
||||
DefaultValue: "true",
|
||||
Validate: validateBoolSetting(rememberLibraryPageStateSettingKey),
|
||||
},
|
||||
"player.hdr_enabled": {
|
||||
Scope: scopeDevice,
|
||||
DefaultValue: "true",
|
||||
@@ -358,6 +373,7 @@ func (h *SettingsHandler) HandleGetDeviceSetting(w http.ResponseWriter, r *http.
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to access user store")
|
||||
return
|
||||
}
|
||||
registerRequestDevice(r.Context(), store, profileID, device)
|
||||
|
||||
value, err := store.GetDeviceSetting(r.Context(), profileID, device.DeviceID, key)
|
||||
if err != nil {
|
||||
@@ -457,6 +473,7 @@ func (h *SettingsHandler) HandleDeleteDeviceSetting(w http.ResponseWriter, r *ht
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to access user store")
|
||||
return
|
||||
}
|
||||
registerRequestDevice(r.Context(), store, profileID, device)
|
||||
|
||||
if err := store.DeleteDeviceSetting(r.Context(), profileID, device.DeviceID, key); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete device setting")
|
||||
@@ -486,6 +503,7 @@ func (h *SettingsHandler) HandleGetEffectiveSettings(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
device := deviceMetadataFromRequest(r)
|
||||
registerRequestDevice(r.Context(), store, profileID, device)
|
||||
keys := parseSettingKeys(keysParam)
|
||||
resp := effectiveSettingsResponse{
|
||||
Settings: make([]effectiveSettingResponse, 0, len(keys)),
|
||||
@@ -516,6 +534,7 @@ func (h *SettingsHandler) HandleGetEffectiveSubtitleAppearance(w http.ResponseWr
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to access user store")
|
||||
return
|
||||
}
|
||||
registerRequestDevice(r.Context(), store, profileID, device)
|
||||
|
||||
resolved, err := h.resolveEffectiveSetting(r.Context(), store, profileID, device, subtitleAppearanceSettingKey)
|
||||
if err != nil {
|
||||
@@ -576,6 +595,36 @@ func deviceMetadataFromRequest(r *http.Request) requestDeviceMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
func registerRequestDevice(
|
||||
ctx context.Context,
|
||||
store userstore.UserStore,
|
||||
profileID string,
|
||||
device requestDeviceMetadata,
|
||||
) {
|
||||
if strings.TrimSpace(profileID) == "" || strings.TrimSpace(device.DeviceID) == "" {
|
||||
return
|
||||
}
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
registry, ok := store.(userstore.DeviceRegistry)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := registry.RegisterDevice(ctx, userstore.DeviceEntry{
|
||||
ProfileID: profileID,
|
||||
DeviceID: device.DeviceID,
|
||||
DeviceName: device.DeviceName,
|
||||
DevicePlatform: device.DevicePlatform,
|
||||
}); err != nil {
|
||||
slog.Warn("failed to register request device",
|
||||
"profile_id", profileID,
|
||||
"device_id", device.DeviceID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func clampHeaderValue(value string, maxLen int) string {
|
||||
value = strings.TrimSpace(value)
|
||||
runes := []rune(value)
|
||||
|
||||
@@ -47,6 +47,14 @@ func (r testAdminUserRepo) GetByID(_ context.Context, id int) (*models.User, err
|
||||
return r.users[id], nil
|
||||
}
|
||||
|
||||
func TestRegisterRequestDeviceNilStore(t *testing.T) {
|
||||
registerRequestDevice(context.Background(), nil, "profile-1", requestDeviceMetadata{
|
||||
DeviceID: "device-1",
|
||||
DeviceName: "Living Room",
|
||||
DevicePlatform: "web",
|
||||
})
|
||||
}
|
||||
|
||||
type mappedTestUserStoreProvider struct {
|
||||
stores map[int]userstore.UserStore
|
||||
}
|
||||
@@ -176,7 +184,7 @@ func TestGetEffectiveSettingsResolvesUserDeviceAndDefaultSources(t *testing.T) {
|
||||
handler := NewSettingsHandler(testUserStoreProvider{store: store})
|
||||
req := httptest.NewRequest(
|
||||
http.MethodGet,
|
||||
"/settings/effective?keys=playback.preferred_quality,player.playback_speed,player.hdr_enabled",
|
||||
"/settings/effective?keys=playback.preferred_quality,player.playback_speed,player.hdr_enabled,ui.remember_library_page_state",
|
||||
nil,
|
||||
)
|
||||
req.Header.Set(deviceIDHeader, "apple-tv")
|
||||
@@ -192,7 +200,7 @@ func TestGetEffectiveSettingsResolvesUserDeviceAndDefaultSources(t *testing.T) {
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(resp.Settings) != 3 {
|
||||
if len(resp.Settings) != 4 {
|
||||
t.Fatalf("settings len = %d", len(resp.Settings))
|
||||
}
|
||||
byKey := make(map[string]effectiveSettingResponse, len(resp.Settings))
|
||||
@@ -208,6 +216,9 @@ func TestGetEffectiveSettingsResolvesUserDeviceAndDefaultSources(t *testing.T) {
|
||||
if got := byKey["player.hdr_enabled"]; got.EffectiveValue != "true" || got.Source != "default" {
|
||||
t.Fatalf("hdr_enabled = %#v", got)
|
||||
}
|
||||
if got := byKey[rememberLibraryPageStateSettingKey]; got.EffectiveValue != "true" || got.Source != "default" {
|
||||
t.Fatalf("remember_library_page_state = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenericSettingsRejectInvalidRegisteredValues(t *testing.T) {
|
||||
@@ -230,6 +241,129 @@ func TestGenericSettingsRejectInvalidRegisteredValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryPageStateIsDeviceScopedJSONSetting(t *testing.T) {
|
||||
store := newProfileTestStore(t)
|
||||
handler := NewSettingsHandler(testUserStoreProvider{store: store})
|
||||
|
||||
req := httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/settings/device/ui.library_page_state",
|
||||
bytes.NewBufferString(`{"value":"{\"version\":1,\"libraries\":{\"7\":{\"search\":\"tab=library\"}}}"}`),
|
||||
)
|
||||
req = withRouteParams(req, map[string]string{"key": libraryPageStateSettingKey})
|
||||
req.Header.Set(deviceIDHeader, "browser")
|
||||
req = req.WithContext(apimw.SetProfileID(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7}), "profile-1"))
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleSetDeviceSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
entry, err := store.GetDeviceSetting(context.Background(), "profile-1", "browser", libraryPageStateSettingKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDeviceSetting: %v", err)
|
||||
}
|
||||
if entry == nil || !strings.Contains(entry.Value, `"tab=library"`) {
|
||||
t.Fatalf("entry = %#v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryPageStateRejectsInvalidJSONAndUserScope(t *testing.T) {
|
||||
store := newProfileTestStore(t)
|
||||
handler := NewSettingsHandler(testUserStoreProvider{store: store})
|
||||
|
||||
req := httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/settings/device/ui.library_page_state",
|
||||
bytes.NewBufferString(`{"value":"not json"}`),
|
||||
)
|
||||
req = withRouteParams(req, map[string]string{"key": libraryPageStateSettingKey})
|
||||
req.Header.Set(deviceIDHeader, "browser")
|
||||
req = req.WithContext(apimw.SetProfileID(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7}), "profile-1"))
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleSetDeviceSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("invalid JSON status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/settings/ui.library_page_state",
|
||||
bytes.NewBufferString(`{"value":"{\"version\":1,\"libraries\":{}}"}`),
|
||||
)
|
||||
req = withRouteParams(req, map[string]string{"key": libraryPageStateSettingKey})
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7}))
|
||||
rec = httptest.NewRecorder()
|
||||
|
||||
handler.HandleSetSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("user-scope status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRememberLibraryPageStateIsDeviceScopedBoolSetting(t *testing.T) {
|
||||
store := newProfileTestStore(t)
|
||||
handler := NewSettingsHandler(testUserStoreProvider{store: store})
|
||||
|
||||
req := httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/settings/device/ui.remember_library_page_state",
|
||||
bytes.NewBufferString(`{"value":"false"}`),
|
||||
)
|
||||
req = withRouteParams(req, map[string]string{"key": rememberLibraryPageStateSettingKey})
|
||||
req.Header.Set(deviceIDHeader, "browser")
|
||||
req = req.WithContext(apimw.SetProfileID(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7}), "profile-1"))
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleSetDeviceSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
entry, err := store.GetDeviceSetting(context.Background(), "profile-1", "browser", rememberLibraryPageStateSettingKey)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDeviceSetting: %v", err)
|
||||
}
|
||||
if entry == nil || entry.Value != "false" {
|
||||
t.Fatalf("entry = %#v", entry)
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/settings/device/ui.remember_library_page_state",
|
||||
bytes.NewBufferString(`{"value":"maybe"}`),
|
||||
)
|
||||
req = withRouteParams(req, map[string]string{"key": rememberLibraryPageStateSettingKey})
|
||||
req.Header.Set(deviceIDHeader, "browser")
|
||||
req = req.WithContext(apimw.SetProfileID(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7}), "profile-1"))
|
||||
rec = httptest.NewRecorder()
|
||||
|
||||
handler.HandleSetDeviceSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("invalid bool status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/settings/ui.remember_library_page_state",
|
||||
bytes.NewBufferString(`{"value":"false"}`),
|
||||
)
|
||||
req = withRouteParams(req, map[string]string{"key": rememberLibraryPageStateSettingKey})
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7}))
|
||||
rec = httptest.NewRecorder()
|
||||
|
||||
handler.HandleSetSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("user-scope status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminCanResetSubtitleAppearanceDeviceOverrides(t *testing.T) {
|
||||
store := newProfileTestStore(t)
|
||||
for _, deviceID := range []string{"apple-tv", "iphone"} {
|
||||
@@ -346,6 +480,18 @@ func TestAdminCanListAndInspectDevicesAcrossUsers(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("SetDeviceSetting store1 second: %v", err)
|
||||
}
|
||||
store1Registry, ok := store1.(userstore.DeviceRegistry)
|
||||
if !ok {
|
||||
t.Fatalf("store1 does not support device registry")
|
||||
}
|
||||
if err := store1Registry.RegisterDevice(context.Background(), userstore.DeviceEntry{
|
||||
ProfileID: "profile-1",
|
||||
DeviceID: "bedroom",
|
||||
DeviceName: "Bedroom TV",
|
||||
DevicePlatform: "Android TV",
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterDevice store1: %v", err)
|
||||
}
|
||||
if err := store2.SetDeviceSetting(context.Background(), userstore.DeviceSettingEntry{
|
||||
ProfileID: "profile-1",
|
||||
DeviceID: "phone",
|
||||
@@ -382,8 +528,21 @@ func TestAdminCanListAndInspectDevicesAcrossUsers(t *testing.T) {
|
||||
if err := json.NewDecoder(rec.Body).Decode(&listResp); err != nil {
|
||||
t.Fatalf("decode list: %v", err)
|
||||
}
|
||||
if len(listResp.Devices) != 2 {
|
||||
t.Fatalf("devices len = %d, want 2", len(listResp.Devices))
|
||||
if len(listResp.Devices) != 3 {
|
||||
t.Fatalf("devices len = %d, want 3", len(listResp.Devices))
|
||||
}
|
||||
var bedroom *adminDeviceSummaryResponse
|
||||
for i := range listResp.Devices {
|
||||
if listResp.Devices[i].DeviceID == "bedroom" {
|
||||
bedroom = &listResp.Devices[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if bedroom == nil {
|
||||
t.Fatalf("registered device without overrides missing: %#v", listResp.Devices)
|
||||
}
|
||||
if bedroom.OverrideCount != 0 || bedroom.ProfileCount != 1 || bedroom.DeviceName != "Bedroom TV" {
|
||||
t.Fatalf("registered device summary = %#v", bedroom)
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodGet, "/admin/devices/7/living-room", nil)
|
||||
@@ -403,6 +562,26 @@ func TestAdminCanListAndInspectDevicesAcrossUsers(t *testing.T) {
|
||||
if len(detailResp.Settings) != 2 {
|
||||
t.Fatalf("detail settings len = %d, want 2", len(detailResp.Settings))
|
||||
}
|
||||
|
||||
req = httptest.NewRequest(http.MethodGet, "/admin/devices/7/bedroom", nil)
|
||||
req = withRouteParams(req, map[string]string{"user_id": "7", "device_id": "bedroom"})
|
||||
rec = httptest.NewRecorder()
|
||||
handler.HandleGetDevice(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("registered detail status = %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if err := json.NewDecoder(rec.Body).Decode(&detailResp); err != nil {
|
||||
t.Fatalf("decode registered detail: %v", err)
|
||||
}
|
||||
if detailResp.DeviceName != "Bedroom TV" || detailResp.OverrideCount != 0 {
|
||||
t.Fatalf("registered detail response = %#v", detailResp)
|
||||
}
|
||||
if len(detailResp.Settings) != 0 {
|
||||
t.Fatalf("registered detail settings len = %d, want 0", len(detailResp.Settings))
|
||||
}
|
||||
if len(detailResp.Profiles) != 1 || detailResp.Profiles[0].ProfileID != "profile-1" {
|
||||
t.Fatalf("registered detail profiles = %#v", detailResp.Profiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminCanResetAllOverridesForOneDevice(t *testing.T) {
|
||||
@@ -433,6 +612,24 @@ func TestAdminCanResetAllOverridesForOneDevice(t *testing.T) {
|
||||
if len(entries) != 1 || entries[0].DeviceID != "phone" {
|
||||
t.Fatalf("entries after delete = %#v", entries)
|
||||
}
|
||||
registry, ok := store.(userstore.DeviceRegistry)
|
||||
if !ok {
|
||||
t.Fatalf("store does not support device registry")
|
||||
}
|
||||
devices, err := registry.ListDevices(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListDevices: %v", err)
|
||||
}
|
||||
foundLivingRoom := false
|
||||
for _, device := range devices {
|
||||
if device.ProfileID == "profile-1" && device.DeviceID == "living-room" {
|
||||
foundLivingRoom = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundLivingRoom {
|
||||
t.Fatalf("registry devices after delete = %#v", devices)
|
||||
}
|
||||
}
|
||||
|
||||
func withRouteParams(req *http.Request, params map[string]string) *http.Request {
|
||||
|
||||
@@ -2,6 +2,7 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -262,6 +263,86 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Subtitle track not found")
|
||||
}
|
||||
|
||||
// HandleSubtitleFonts extracts embedded container font attachments for ASS/SSA
|
||||
// playback. The web player loads these bytes into JASSUB before creating the
|
||||
// renderer so libass can resolve script font names deterministically.
|
||||
func (h *StreamHandler) HandleSubtitleFonts(w http.ResponseWriter, r *http.Request) {
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
if userID == 0 {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
|
||||
return
|
||||
}
|
||||
|
||||
sessionID := chi.URLParam(r, "session_id")
|
||||
if sessionID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Session ID is required")
|
||||
return
|
||||
}
|
||||
setPlaybackSessionLogContext(r, sessionID)
|
||||
|
||||
session, err := h.sessionMgr.GetSession(sessionID)
|
||||
if err != nil {
|
||||
writePlaybackSessionNotFound(w)
|
||||
return
|
||||
}
|
||||
if session.UserID != userID {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user")
|
||||
return
|
||||
}
|
||||
|
||||
file, err := h.fileResolver.GetByID(r.Context(), session.MediaFileID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
return
|
||||
}
|
||||
if file == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
return
|
||||
}
|
||||
if err := preflightPlaybackFile(r.Context(), file, h.MissingMarker, h.EventsHub); err != nil {
|
||||
if isPlaybackFileMissing(err) {
|
||||
h.abortPlaybackSession(r.Context(), session)
|
||||
}
|
||||
writePlaybackFilePreflightError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
trackParam := chi.URLParam(r, "track")
|
||||
trackIndex, _, err := playback.ParseSubtitleTrackParam(trackParam)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid subtitle track index")
|
||||
return
|
||||
}
|
||||
|
||||
embeddedIndex := trackIndex - len(file.ExternalSubtitles)
|
||||
if embeddedIndex < 0 || embeddedIndex >= len(file.SubtitleTracks) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Embedded subtitle track not found")
|
||||
return
|
||||
}
|
||||
if !playback.IsASS(file.SubtitleTracks[embeddedIndex].Codec) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Subtitle font bundles are only available for ASS/SSA tracks")
|
||||
return
|
||||
}
|
||||
|
||||
fonts, err := playback.ExtractAttachedSubtitleFonts(r.Context(), file.FilePath, h.FFmpegPath)
|
||||
if err != nil {
|
||||
slog.WarnContext(r.Context(), "subtitle font extraction failed",
|
||||
"file_id", file.ID,
|
||||
"track", trackIndex,
|
||||
"error", err,
|
||||
)
|
||||
writeError(w, http.StatusInternalServerError, "font_extract_failed", "Failed to extract subtitle fonts")
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
if err := json.NewEncoder(w).Encode(playback.EncodeSubtitleFontBundle(fonts)); err != nil {
|
||||
slog.WarnContext(r.Context(), "subtitle font response encode failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *StreamHandler) syncSessionsNow(ctx context.Context, reason string) {
|
||||
if h == nil || h.SessionSyncer == nil {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/ai"
|
||||
)
|
||||
|
||||
// SubtitleAIHandler exposes on-demand AI subtitle translation backed by the
|
||||
// configured OpenAI-compatible engine. Generated tracks are stored as ordinary
|
||||
// downloaded subtitles, so they reach every client through the existing
|
||||
// subtitle pipeline.
|
||||
type SubtitleAIHandler struct {
|
||||
service *ai.Service
|
||||
FileAuthorizer *MediaFileAuthorizer
|
||||
}
|
||||
|
||||
// NewSubtitleAIHandler creates a handler backed by the given service.
|
||||
func NewSubtitleAIHandler(service *ai.Service) *SubtitleAIHandler {
|
||||
return &SubtitleAIHandler{service: service}
|
||||
}
|
||||
|
||||
// authorizeMediaFileAccess verifies the caller may access the given media file.
|
||||
// Shared by the subtitle handlers so authorization stays in one place.
|
||||
func authorizeMediaFileAccess(w http.ResponseWriter, r *http.Request, authorizer *MediaFileAuthorizer, fileID int) bool {
|
||||
if authorizer == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Media file authorization is not configured")
|
||||
return false
|
||||
}
|
||||
if _, err := authorizer.Authorize(r, fileID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, catalog.ErrItemNotFound), errors.Is(err, catalog.ErrEpisodeNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type translateSubtitleRequest struct {
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
SourceIndex int `json:"source_index"`
|
||||
SourceLanguage string `json:"source_language"`
|
||||
TargetLanguage string `json:"target_language"`
|
||||
SessionID string `json:"session_id"`
|
||||
StartPosition float64 `json:"start_position"`
|
||||
}
|
||||
|
||||
// HandleStatus reports whether AI subtitle translation is available, so the
|
||||
// player can show or hide the entry point. GET /api/v1/subtitles/ai/status
|
||||
func (h *SubtitleAIHandler) HandleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"enabled": h.service.Enabled()})
|
||||
}
|
||||
|
||||
// WriteSubtitleAIDisabledStatus answers the AI status capability probe with a
|
||||
// 200 {"enabled": false} when no AI handler is wired, so the client gets a clean
|
||||
// negative instead of a 404 (the 2-segment /ai/status path is not shadowed by the
|
||||
// 1-segment /{media_file_id} route — they never compete in chi's router).
|
||||
func WriteSubtitleAIDisabledStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"enabled": false})
|
||||
}
|
||||
|
||||
// HandleTranslate enqueues a translation job. POST /api/v1/subtitles/ai/translate
|
||||
func (h *SubtitleAIHandler) HandleTranslate(w http.ResponseWriter, r *http.Request) {
|
||||
var req translateSubtitleRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.MediaFileID <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "media_file_id is required")
|
||||
return
|
||||
}
|
||||
if req.TargetLanguage == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "target_language is required")
|
||||
return
|
||||
}
|
||||
|
||||
if !authorizeMediaFileAccess(w, r, h.FileAuthorizer, req.MediaFileID) {
|
||||
return
|
||||
}
|
||||
|
||||
var requestedBy *int
|
||||
if userID := apimw.GetUserID(r.Context()); userID != 0 {
|
||||
requestedBy = &userID
|
||||
}
|
||||
|
||||
job, err := h.service.Enqueue(r.Context(), ai.JobRequest{
|
||||
MediaFileID: req.MediaFileID,
|
||||
Kind: ai.JobKindTranslate,
|
||||
SourceIndex: req.SourceIndex,
|
||||
SourceLanguage: req.SourceLanguage,
|
||||
TargetLanguage: req.TargetLanguage,
|
||||
RequestedBy: requestedBy,
|
||||
SessionID: req.SessionID,
|
||||
StartPosition: req.StartPosition,
|
||||
})
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, ai.ErrEngineNotConfigured):
|
||||
writeError(w, http.StatusServiceUnavailable, "not_configured",
|
||||
"AI subtitle translation is not configured on this server")
|
||||
case errors.Is(err, ai.ErrInvalidRequest):
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
default:
|
||||
slog.Error("failed to enqueue subtitle translation",
|
||||
"media_file_id", req.MediaFileID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to start translation")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job})
|
||||
}
|
||||
|
||||
// HandleGetJob returns a job's current state. GET /api/v1/subtitles/ai/jobs/{job_id}
|
||||
func (h *SubtitleAIHandler) HandleGetJob(w http.ResponseWriter, r *http.Request) {
|
||||
job, ok := h.loadAuthorizedJob(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"job": job})
|
||||
}
|
||||
|
||||
// HandleCancelJob cancels a job. POST /api/v1/subtitles/ai/jobs/{job_id}/cancel
|
||||
func (h *SubtitleAIHandler) HandleCancelJob(w http.ResponseWriter, r *http.Request) {
|
||||
job, ok := h.loadAuthorizedJob(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := h.service.Cancel(r.Context(), job.ID); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel job")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// HandleListJobs lists recent jobs for a media file.
|
||||
// GET /api/v1/subtitles/ai/jobs?media_file_id=N
|
||||
func (h *SubtitleAIHandler) HandleListJobs(w http.ResponseWriter, r *http.Request) {
|
||||
mediaFileID, err := strconv.Atoi(r.URL.Query().Get("media_file_id"))
|
||||
if err != nil || mediaFileID <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid_id", "Invalid or missing media_file_id")
|
||||
return
|
||||
}
|
||||
if !authorizeMediaFileAccess(w, r, h.FileAuthorizer, mediaFileID) {
|
||||
return
|
||||
}
|
||||
jobs, err := h.service.ListJobs(r.Context(), mediaFileID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "list_error", "Failed to list jobs")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"jobs": jobs})
|
||||
}
|
||||
|
||||
// loadAuthorizedJob parses the job_id param, loads the job, and authorizes
|
||||
// access against its media file. It writes the error response on failure.
|
||||
func (h *SubtitleAIHandler) loadAuthorizedJob(w http.ResponseWriter, r *http.Request) (*ai.Job, bool) {
|
||||
id, err := strconv.ParseInt(chi.URLParam(r, "job_id"), 10, 64)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_id", "Invalid job ID")
|
||||
return nil, false
|
||||
}
|
||||
job, err := h.service.GetJob(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ai.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return nil, false
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load job")
|
||||
return nil, false
|
||||
}
|
||||
if !authorizeMediaFileAccess(w, r, h.FileAuthorizer, job.MediaFileID) {
|
||||
return nil, false
|
||||
}
|
||||
return job, true
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"strings"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
@@ -96,20 +95,7 @@ type downloadSubtitleRequest struct {
|
||||
}
|
||||
|
||||
func (h *SubtitleSearchHandler) authorizeMediaFile(w http.ResponseWriter, r *http.Request, fileID int) bool {
|
||||
if h.FileAuthorizer == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Media file authorization is not configured")
|
||||
return false
|
||||
}
|
||||
if _, err := h.FileAuthorizer.Authorize(r, fileID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, catalog.ErrItemNotFound), errors.Is(err, catalog.ErrEpisodeNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
return authorizeMediaFileAccess(w, r, h.FileAuthorizer, fileID)
|
||||
}
|
||||
|
||||
// HandleSearch handles POST /api/v1/subtitles/search
|
||||
|
||||
@@ -7,26 +7,41 @@ import (
|
||||
)
|
||||
|
||||
type userStateEventPayload struct {
|
||||
ProfileID string `json:"profile_id"`
|
||||
ContentID string `json:"content_id,omitempty"`
|
||||
SeriesID string `json:"series_id,omitempty"`
|
||||
Change string `json:"change"`
|
||||
ProfileID string `json:"profile_id"`
|
||||
ContentID string `json:"content_id,omitempty"`
|
||||
SeriesID string `json:"series_id,omitempty"`
|
||||
Change string `json:"change"`
|
||||
Played *bool `json:"played,omitempty"`
|
||||
IsFavorite *bool `json:"is_favorite,omitempty"`
|
||||
InWatchlist *bool `json:"in_watchlist,omitempty"`
|
||||
}
|
||||
|
||||
type userStateEventState struct {
|
||||
Played *bool
|
||||
IsFavorite *bool
|
||||
InWatchlist *bool
|
||||
}
|
||||
|
||||
const userStateChangedEvent = "user_state.changed"
|
||||
|
||||
func publishUserStateEvent(
|
||||
ctx context.Context,
|
||||
hub *evt.Hub,
|
||||
userID int,
|
||||
profileID, contentID, seriesID, change, eventName string,
|
||||
profileID, contentID, seriesID, change string,
|
||||
state userStateEventState,
|
||||
) {
|
||||
if hub == nil || userID == 0 || profileID == "" || change == "" || eventName == "" {
|
||||
if hub == nil || userID == 0 || profileID == "" || change == "" {
|
||||
return
|
||||
}
|
||||
_ = hub.PublishJSON(ctx, evt.ChannelUserState, eventName, userStateEventPayload{
|
||||
ProfileID: profileID,
|
||||
ContentID: contentID,
|
||||
SeriesID: seriesID,
|
||||
Change: change,
|
||||
_ = hub.PublishJSON(ctx, evt.ChannelUserState, userStateChangedEvent, userStateEventPayload{
|
||||
ProfileID: profileID,
|
||||
ContentID: contentID,
|
||||
SeriesID: seriesID,
|
||||
Change: change,
|
||||
Played: state.Played,
|
||||
IsFavorite: state.IsFavorite,
|
||||
InWatchlist: state.InWatchlist,
|
||||
}, evt.PublishOptions{
|
||||
UserID: userID,
|
||||
ProfileID: profileID,
|
||||
|
||||
@@ -547,8 +547,8 @@ func requestBaseURL(r *http.Request) string {
|
||||
scheme = forwarded
|
||||
}
|
||||
host := r.Host
|
||||
if forwardedHost := strings.TrimSpace(r.Header.Get("X-Forwarded-Host")); forwardedHost != "" {
|
||||
host = forwardedHost
|
||||
if forwarded := forwardedHost(r); forwarded != "" {
|
||||
host = forwarded
|
||||
}
|
||||
return scheme + "://" + host
|
||||
}
|
||||
|
||||
@@ -28,11 +28,22 @@ func checkWebSocketOrigin(r *http.Request) bool {
|
||||
}
|
||||
|
||||
originURL, err := url.Parse(origin)
|
||||
if err != nil {
|
||||
if err != nil || originURL.Host == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.EqualFold(originURL.Host, r.Host)
|
||||
if strings.EqualFold(originURL.Host, r.Host) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Behind a TLS-terminating CDN/proxy, r.Host is the internal origin host
|
||||
// while the public host the browser used arrives as X-Forwarded-Host. The
|
||||
// browser's Origin reflects that public host, so accept it too.
|
||||
if fwd := forwardedHost(r); fwd != "" && strings.EqualFold(originURL.Host, fwd) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func configureWebSocket(conn *websocket.Conn) {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckWebSocketOrigin(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
host string
|
||||
origin string
|
||||
forwardedHost string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "missing origin is allowed for non-browser clients",
|
||||
host: "origin.internal:8097",
|
||||
origin: "",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "same origin matches host directly",
|
||||
host: "silo.example.com",
|
||||
origin: "https://silo.example.com",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "foreign origin without a proxy is rejected",
|
||||
host: "silo.example.com",
|
||||
origin: "https://evil.example.com",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// The CDN regression: it rewrites Host to the internal origin and
|
||||
// carries the public host in X-Forwarded-Host.
|
||||
name: "cdn-rewritten host accepts the forwarded public origin",
|
||||
host: "origin.internal:8097",
|
||||
origin: "https://silo.example.test",
|
||||
forwardedHost: "silo.example.test",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "foreign origin is rejected even with a forwarded host",
|
||||
host: "origin.internal:8097",
|
||||
origin: "https://evil.example.com",
|
||||
forwardedHost: "silo.example.test",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "first hop of a multi-proxy forwarded host is honored",
|
||||
host: "origin.internal:8097",
|
||||
origin: "https://silo.example.test",
|
||||
forwardedHost: "silo.example.test, edge.internal",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "opaque null origin is rejected",
|
||||
host: "silo.example.com",
|
||||
origin: "null",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "unparseable origin is rejected",
|
||||
host: "silo.example.com",
|
||||
origin: "https://silo.example.com\x7f",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/playback/sessions/abc/control/ws", nil)
|
||||
req.Host = tt.host
|
||||
if tt.origin != "" {
|
||||
req.Header.Set("Origin", tt.origin)
|
||||
}
|
||||
if tt.forwardedHost != "" {
|
||||
req.Header.Set("X-Forwarded-Host", tt.forwardedHost)
|
||||
}
|
||||
|
||||
if got := checkWebSocketOrigin(req); got != tt.want {
|
||||
t.Fatalf("checkWebSocketOrigin() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+158
-11
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/api/handlers"
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/autoscan"
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/catalogseed"
|
||||
@@ -54,6 +55,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/scanqueue"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
subtitleai "github.com/Silo-Server/silo-server/internal/subtitles/ai"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/opensubtitles"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/subdl"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/subsource"
|
||||
@@ -87,11 +89,13 @@ type Dependencies struct {
|
||||
SessionMgr *playback.SessionManager // playback session manager (may be nil)
|
||||
SkippedRootRepo *metadata.SkippedRootRepository // skipped root repository (may be nil)
|
||||
StaleIDRepo *metadata.StaleMediaIDRepository // stale media ID repository (may be nil)
|
||||
Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil)
|
||||
NodeRepo *nodepool.Repository // stream node repository (may be nil)
|
||||
ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil)
|
||||
TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil)
|
||||
SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger
|
||||
MovieMatchQueueRepo *metadata.MovieMatchQueueRepository
|
||||
SeriesRootMatchQueueRepo *metadata.SeriesRootMatchQueueRepository
|
||||
Refresher handlers.AdminMetadataRefresher // metadata refresher (may be nil)
|
||||
NodeRepo *nodepool.Repository // stream node repository (may be nil)
|
||||
ProxyPool *nodepool.ProxyPool // proxy node pool (may be nil)
|
||||
TranscodePool *nodepool.TranscodePool // transcode node pool (may be nil)
|
||||
SessionSyncer handlers.PlaybackSessionSyncer // optional; immediate playback session sync trigger
|
||||
EventBus cache.EventBus
|
||||
AdminStatsProvider handlers.AdminStatsSource
|
||||
Recommender recommendations.Recommender // nil when disabled
|
||||
@@ -114,6 +118,7 @@ type Dependencies struct {
|
||||
FFmpegLogSink playback.FFmpegLogSink
|
||||
RedisClient *redis.Client // for session listing (may be nil)
|
||||
TaskManager *taskmanager.TaskManager // task manager (may be nil)
|
||||
AdminJobCancelRegistry *adminjob.CancelRegistry
|
||||
IntroRepository *intromarkers.Repository
|
||||
IntroAnalyzer *intromarkers.Analyzer
|
||||
MarkerRegistry *markers.Registry
|
||||
@@ -141,6 +146,11 @@ type Dependencies struct {
|
||||
UserCollectionSync *usercollections.Service
|
||||
UserCollectionScheduler *usercollections.Scheduler
|
||||
|
||||
// TrendingRefresher refreshes the persisted trending_discover snapshots.
|
||||
// Built in main.go with TMDB wired; its Trakt fetcher is propagated here in
|
||||
// router.go once the Trakt adapter exists (mirrors UserCollectionSync).
|
||||
TrendingRefresher *sections.TrendingRefresher
|
||||
|
||||
// MDBListClient is used by user-facing list discovery endpoints
|
||||
// (search/top). May be nil; the handlers report "not configured" in
|
||||
// that case rather than failing.
|
||||
@@ -296,6 +306,12 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
libraryHandler.EventsHub = deps.EventsHub
|
||||
libraryHandler.ScanRegistry = deps.ScanRegistry
|
||||
libraryHandler.ScanQueue = deps.LibraryScanQueue
|
||||
libraryHandler.MovieMatchQueueRepo = deps.MovieMatchQueueRepo
|
||||
libraryHandler.SeriesMatchQueueRepo = deps.SeriesRootMatchQueueRepo
|
||||
libraryHandler.RawMatchBacklogRepo = deps.FileRepo
|
||||
if deps.Config != nil {
|
||||
libraryHandler.TVSeriesRootQueue = deps.Config.Matcher.TVSeriesRootQueueEnabled()
|
||||
}
|
||||
if deps.DB != nil {
|
||||
libraryHandler.JobRepo = adminjob.NewRepository(deps.DB)
|
||||
}
|
||||
@@ -350,6 +366,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
var calendarRepo *catalog.CalendarRepository
|
||||
var webhookSyncHandler *handlers.WebhookSyncHandler
|
||||
var requestHandler *handlers.RequestsHandler
|
||||
var autoscanHandler *handlers.AutoscanHandler
|
||||
if deps.DB != nil {
|
||||
browseRepo := catalog.NewBrowseRepository(deps.DB)
|
||||
itemRepo = catalog.NewItemRepository(deps.DB)
|
||||
@@ -413,15 +430,39 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
if deps.Config != nil {
|
||||
tmdbAPIKey = deps.Config.TMDBAPIKey
|
||||
}
|
||||
requestsRepo := mediarequests.NewRepository(deps.DB)
|
||||
requestSvc := mediarequests.NewService(
|
||||
mediarequests.NewRepository(deps.DB),
|
||||
requestsRepo,
|
||||
tmdb.NewClient(tmdbAPIKey, 40),
|
||||
mediarequests.NewCatalogPresence(itemRepo, providerIDRepo),
|
||||
)
|
||||
requestSvc.SetSecretResolver(settingsRepo)
|
||||
requestSvc.SetFulfillmentAdapters(radarr.NewClient(nil), sonarr.NewClient(nil))
|
||||
if viewerResolver != nil {
|
||||
requestSvc.SetEntitlementResolver(mediarequests.NewAccessEntitlements(viewerResolver))
|
||||
}
|
||||
requestHandler = handlers.NewRequestsHandler(requestSvc)
|
||||
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB)
|
||||
if deps.FolderRepo != nil && deps.LibraryScanQueue != nil && deps.PluginService != nil {
|
||||
autoscanSvc := BuildAutoscanService(
|
||||
autoscanRepo,
|
||||
deps.PluginService,
|
||||
plugins.NewInstallationStore(deps.DB),
|
||||
requestsRepo,
|
||||
settingsRepo,
|
||||
deps.FolderRepo,
|
||||
deps.LibraryScanQueue,
|
||||
deps.RedisClient,
|
||||
)
|
||||
autoscanHandler = handlers.NewAutoscanHandler(autoscanRepo, autoscanSvc)
|
||||
// Wire the optional poll-task rescheduler so a settings change
|
||||
// re-applies the poll interval without a restart.
|
||||
if deps.TaskManager != nil {
|
||||
autoscanHandler.SetTriggerUpdater(deps.TaskManager)
|
||||
}
|
||||
}
|
||||
|
||||
if deps.PersonRepo != nil {
|
||||
peopleHandler = handlers.NewPeopleHandler(deps.PersonRepo, browseRepo, itemRepo, detailSvc)
|
||||
peopleHandler.SetItemsHandler(itemsHandler)
|
||||
@@ -483,6 +524,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
settingsHandler.SetServerSettings(settingsRepo)
|
||||
}
|
||||
homeDismissalHandler = handlers.NewHomeDismissalHandler(deps.UserStoreProvider)
|
||||
homeDismissalHandler.EventsHub = deps.EventsHub
|
||||
subtitlePrefHandler = handlers.NewSubtitlePrefHandler(deps.UserStoreProvider)
|
||||
audioPrefHandler = handlers.NewAudioPrefHandler(deps.UserStoreProvider)
|
||||
libraryPlaybackPrefHandler = handlers.NewLibraryPlaybackPrefHandler(deps.UserStoreProvider)
|
||||
@@ -526,6 +568,11 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
subtitleRepo = subtitles.NewPgRepository(deps.DB)
|
||||
}
|
||||
|
||||
// Notifier that pushes "subtitle ready" events to active sessions when an AI
|
||||
// translation completes. Assigned inside the playback handler block where the
|
||||
// realtime hub and session manager are in scope; nil when playback is off.
|
||||
var subtitleAINotifier *playback.SubtitleReadyNotifier
|
||||
|
||||
// Build playback handler if session manager is available.
|
||||
var playbackHandler *handlers.PlaybackHandler
|
||||
var adminPlaybackControlHandler *handlers.AdminPlaybackControlHandler
|
||||
@@ -620,6 +667,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
playbackHandler.MarkerUpserter = deps.FileRepo
|
||||
}
|
||||
playbackHandler.MarkerUpdateNotifier = playback.NewMarkerUpdateNotifier(deps.SessionMgr, realtimeHub)
|
||||
subtitleAINotifier = playback.NewSubtitleReadyNotifier(deps.SessionMgr, realtimeHub)
|
||||
adminPlaybackControlHandler = handlers.NewAdminPlaybackControlHandler(playbackHandler)
|
||||
|
||||
if deps.DB != nil && deps.FileRepo != nil && viewerResolver != nil && deps.Config != nil && detailSvc != nil {
|
||||
@@ -684,6 +732,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
catalogSeedHandler = handlers.NewCatalogSeedHandler(catalogseed.NewService(deps.DB, deps.PersonRepo, recommendations.NewRepo(deps.DB)), jobRepo, privateStore)
|
||||
catalogSeedHandler.RealtimeHub = deps.RealtimeHub
|
||||
adminJobsHandler = handlers.NewAdminJobsHandler(jobRepo, privateStore)
|
||||
adminJobsHandler.CancelRegistry = deps.AdminJobCancelRegistry
|
||||
adminJobsHandler.RealtimeHub = deps.RealtimeHub
|
||||
if adminHandler != nil && deps.FolderRepo != nil && deps.FileRepo != nil && itemRepo != nil && episodeRepo != nil {
|
||||
adminHandler.JobRepo = jobRepo
|
||||
adminHandler.ItemRefreshResolver = adminjob.NewItemRefreshResolver(
|
||||
@@ -783,6 +833,40 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
adminSubtitleHandler.SetDownloadedSubtitleDeps(deps.DB, subtitleManager)
|
||||
}
|
||||
|
||||
// Build the AI subtitle handler (on-demand translation). Generated tracks are
|
||||
// stored as ordinary downloaded subtitles, so they reach every client through
|
||||
// the existing subtitle pipeline with no client changes.
|
||||
var subtitleAIHandler *handlers.SubtitleAIHandler
|
||||
if subtitleManager != nil && subtitleRepo != nil && deps.FileRepo != nil && deps.DB != nil && deps.Config != nil {
|
||||
aiCfg := subtitleai.Config{
|
||||
Enabled: deps.Config.SubtitleAI.Enabled,
|
||||
BaseURL: deps.Config.SubtitleAI.BaseURL,
|
||||
APIKey: deps.Config.SubtitleAI.APIKey,
|
||||
ChatModel: deps.Config.SubtitleAI.ChatModel,
|
||||
MaxConcurrentJobs: deps.Config.SubtitleAI.MaxConcurrentJobs,
|
||||
BatchSize: deps.Config.SubtitleAI.BatchSize,
|
||||
ContextNeighbors: deps.Config.SubtitleAI.ContextNeighbors,
|
||||
}
|
||||
var aiNotifier subtitleai.Notifier
|
||||
if subtitleAINotifier != nil {
|
||||
aiNotifier = subtitleAINotifier
|
||||
}
|
||||
aiService := subtitleai.NewService(
|
||||
deps.AppContext,
|
||||
aiCfg,
|
||||
subtitleai.NewPgJobRepository(deps.DB),
|
||||
subtitleai.NewLLMTranslator(subtitleai.NewClient(aiCfg), aiCfg.BatchSize, aiCfg.ContextNeighbors),
|
||||
subtitleManager,
|
||||
subtitleRepo,
|
||||
deps.FileRepo,
|
||||
aiNotifier,
|
||||
deps.Config.Playback.FFmpegPath,
|
||||
slog.Default(),
|
||||
)
|
||||
aiService.Recover()
|
||||
subtitleAIHandler = handlers.NewSubtitleAIHandler(aiService)
|
||||
}
|
||||
|
||||
// Build section handler if DB is available.
|
||||
var sectionHandler *handlers.SectionHandler
|
||||
var sectionSettingsHandler *handlers.SectionSettingsHandler
|
||||
@@ -894,6 +978,18 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
}
|
||||
|
||||
// Propagate the now-wired Trakt fetcher to the trending refresher (built
|
||||
// in main.go with TMDB only, before the Trakt adapter existed).
|
||||
if deps.TrendingRefresher != nil && deps.TrendingRefresher.TraktTrending == nil {
|
||||
deps.TrendingRefresher.TraktTrending = libraryCollectionService.TraktCollections
|
||||
}
|
||||
|
||||
// Wire the trending snapshot reader into the section fetcher. The
|
||||
// trending_discover home section reads its list from the persisted
|
||||
// snapshot table; the upstream fetch happens out-of-band in the refresh
|
||||
// task, so the read path never calls the provider.
|
||||
sectionFetcher.TrendingSnapshots = sections.NewTrendingSnapshotRepository(deps.DB)
|
||||
|
||||
libraryCollectionHandler = handlers.NewLibraryCollectionHandler(
|
||||
libraryCollectionRepo,
|
||||
libraryCollectionService,
|
||||
@@ -1124,6 +1220,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
StateTTL: 10 * time.Minute,
|
||||
})
|
||||
}
|
||||
authHandler.SetOAuthRoutesAvailable(oauthHandler != nil)
|
||||
|
||||
r.Route("/auth", func(r chi.Router) {
|
||||
if deps.RateLimitMW != nil {
|
||||
@@ -1237,12 +1334,16 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/stale-ids", libraryHandler.HandleListStaleIDs)
|
||||
r.Post("/stale-ids/{contentID}/rematch", libraryHandler.HandleRematchStaleID)
|
||||
r.Get("/unmatched-items", libraryHandler.HandleListUnmatchedItems)
|
||||
r.Get("/metadata-match-queue", libraryHandler.HandleListMetadataMatchQueues)
|
||||
r.Post("/", libraryHandler.HandleCreateLibrary)
|
||||
r.Put("/reorder", libraryHandler.HandleReorderLibraries)
|
||||
r.Put("/{id}", libraryHandler.HandleUpdateLibrary)
|
||||
r.Delete("/{id}", libraryHandler.HandleDeleteLibrary)
|
||||
r.Post("/{id}/check-mount", libraryHandler.HandleCheckLibraryMount)
|
||||
r.Post("/{id}/confirm-empty-root-cleanup", libraryHandler.HandleConfirmEmptyRootCleanup)
|
||||
r.Get("/{id}/metadata-match-queue", libraryHandler.HandleGetMetadataMatchQueue)
|
||||
r.Post("/{id}/metadata-match-queue/retry", libraryHandler.HandleRetryMetadataMatchQueue)
|
||||
r.Post("/{id}/metadata-match-queue/cancel", libraryHandler.HandleCancelMetadataMatchQueue)
|
||||
r.Post("/{id}/refresh-metadata", libraryHandler.HandleRefreshLibraryMetadata)
|
||||
r.Get("/{id}/providers", libraryHandler.HandleGetLibraryProviders)
|
||||
r.Put("/{id}/providers", libraryHandler.HandleSetLibraryProviders)
|
||||
@@ -1273,7 +1374,9 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
|
||||
if calendarRepo != nil {
|
||||
calendarHandler := handlers.NewCalendarHandler(calendarRepo, detailSvc)
|
||||
calendarPopular := recommendations.NewRepo(deps.DB)
|
||||
calendarTrending := sections.NewTrendingSnapshotRepository(deps.DB)
|
||||
calendarHandler := handlers.NewCalendarHandler(calendarRepo, detailSvc, calendarPopular, calendarTrending)
|
||||
r.With(apimw.RequireProfile).Get("/calendar", calendarHandler.HandleGetCalendar)
|
||||
}
|
||||
|
||||
@@ -1544,20 +1647,36 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
})
|
||||
}
|
||||
|
||||
// Subtitle search routes.
|
||||
// Subtitle search + AI translation routes.
|
||||
if subtitleSearchHandler != nil {
|
||||
if deps.FileRepo != nil && itemRepo != nil {
|
||||
subtitleSearchHandler.FileAuthorizer = &handlers.MediaFileAuthorizer{
|
||||
fileAuthorizer := &handlers.MediaFileAuthorizer{
|
||||
FileResolver: deps.FileRepo,
|
||||
ItemAccess: itemRepo,
|
||||
EpisodeLookup: episodeRepo,
|
||||
}
|
||||
subtitleSearchHandler.FileAuthorizer = fileAuthorizer
|
||||
if subtitleAIHandler != nil {
|
||||
subtitleAIHandler.FileAuthorizer = fileAuthorizer
|
||||
}
|
||||
}
|
||||
r.Route("/subtitles", func(r chi.Router) {
|
||||
r.Post("/search", subtitleSearchHandler.HandleSearch)
|
||||
r.Post("/download", subtitleSearchHandler.HandleDownload)
|
||||
r.Post("/upload", subtitleSearchHandler.HandleUpload)
|
||||
r.Post("/detect-language", subtitleSearchHandler.HandleDetectLanguage)
|
||||
if subtitleAIHandler != nil {
|
||||
r.Get("/ai/status", subtitleAIHandler.HandleStatus)
|
||||
r.Post("/ai/translate", subtitleAIHandler.HandleTranslate)
|
||||
r.Get("/ai/jobs", subtitleAIHandler.HandleListJobs)
|
||||
r.Get("/ai/jobs/{job_id}", subtitleAIHandler.HandleGetJob)
|
||||
r.Post("/ai/jobs/{job_id}/cancel", subtitleAIHandler.HandleCancelJob)
|
||||
} else {
|
||||
// Answer the capability probe with 200 {"enabled": false}
|
||||
// when AI translation isn't wired, so the client gets a
|
||||
// clean negative instead of a 404.
|
||||
r.Get("/ai/status", handlers.WriteSubtitleAIDisabledStatus)
|
||||
}
|
||||
r.Get("/{media_file_id}", subtitleSearchHandler.HandleList)
|
||||
r.Delete("/{id}", subtitleSearchHandler.HandleDelete)
|
||||
})
|
||||
@@ -1618,6 +1737,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/stream/{session_id}", streamHandler.HandleStream)
|
||||
r.Head("/stream/{session_id}", streamHandler.HandleStream)
|
||||
r.Get("/stream/{session_id}/subtitles/{track}", streamHandler.HandleSubtitle)
|
||||
r.Get("/stream/{session_id}/subtitles/{track}/fonts", streamHandler.HandleSubtitleFonts)
|
||||
}
|
||||
|
||||
// Download routes.
|
||||
@@ -1787,6 +1907,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
if adminJobsHandler != nil {
|
||||
r.Route("/jobs", func(r chi.Router) {
|
||||
r.Get("/", adminJobsHandler.HandleList)
|
||||
r.Post("/{id}/cancel", adminJobsHandler.HandleCancel)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1810,6 +1931,10 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/installations", pluginHandler.HandleListInstallations)
|
||||
r.Post("/installations", pluginHandler.HandleCreateInstallation)
|
||||
r.Post("/uploads", pluginHandler.HandleUploadInstallation)
|
||||
r.Post("/uploads/chunked", pluginHandler.HandleCreateChunkedUpload)
|
||||
r.Put("/uploads/chunked/{upload_id}/chunks/{chunk_index}", pluginHandler.HandleUploadChunk)
|
||||
r.Post("/uploads/chunked/{upload_id}/complete", pluginHandler.HandleCompleteChunkedUpload)
|
||||
r.Delete("/uploads/chunked/{upload_id}", pluginHandler.HandleCancelChunkedUpload)
|
||||
r.Put("/installations/{id}", pluginHandler.HandleUpdateInstallation)
|
||||
r.Post("/installations/{id}/update", pluginHandler.HandleApplyUpdate)
|
||||
r.Post("/installations/{id}/config/test", pluginHandler.HandleTestInstallationConfig)
|
||||
@@ -1998,8 +2123,30 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/request-users/{user_id}/limit", requestHandler.HandleGetUserLimit)
|
||||
r.Put("/request-users/{user_id}/limit", requestHandler.HandleUpdateUserLimit)
|
||||
r.Get("/request-integrations", requestHandler.HandleListIntegrations)
|
||||
r.Put("/request-integrations", requestHandler.HandleUpdateIntegrations)
|
||||
r.Post("/request-integrations/{kind}/options", requestHandler.HandleLoadIntegrationOptions)
|
||||
r.Post("/request-integrations", requestHandler.HandleCreateIntegration)
|
||||
r.Put("/request-integrations/{id}", requestHandler.HandleUpdateIntegration)
|
||||
r.Delete("/request-integrations/{id}", requestHandler.HandleDeleteIntegration)
|
||||
r.Post("/request-integrations/{id}/options", requestHandler.HandleLoadIntegrationOptions)
|
||||
}
|
||||
|
||||
if autoscanHandler != nil {
|
||||
r.Get("/autoscan/settings", autoscanHandler.HandleGetSettings)
|
||||
r.Put("/autoscan/settings", autoscanHandler.HandleUpdateSettings)
|
||||
r.Get("/autoscan/connections", autoscanHandler.HandleListConnections)
|
||||
r.Post("/autoscan/connections", autoscanHandler.HandleCreateConnection)
|
||||
r.Put("/autoscan/connections/{id}", autoscanHandler.HandleUpdateConnection)
|
||||
r.Delete("/autoscan/connections/{id}", autoscanHandler.HandleDeleteConnection)
|
||||
r.Post("/autoscan/connections/test", autoscanHandler.HandleTestConnection)
|
||||
r.Get("/autoscan/scan-source-plugins", autoscanHandler.HandleListAvailableScanSources)
|
||||
r.Get("/autoscan/sources", autoscanHandler.HandleListSources)
|
||||
r.Post("/autoscan/sources", autoscanHandler.HandleCreateSource)
|
||||
r.Put("/autoscan/sources/{id}", autoscanHandler.HandleUpdateSource)
|
||||
r.Delete("/autoscan/sources/{id}", autoscanHandler.HandleDeleteSource)
|
||||
r.Get("/autoscan/sources/{id}/rewrite-suggestions", autoscanHandler.HandleRewriteSuggestions)
|
||||
r.Get("/autoscan/scans", autoscanHandler.HandleListScans)
|
||||
r.Get("/autoscan/events", autoscanHandler.HandleListEvents)
|
||||
r.Post("/autoscan/trigger", autoscanHandler.HandleTrigger)
|
||||
r.Get("/autoscan/status", autoscanHandler.HandleStatus)
|
||||
}
|
||||
|
||||
if deps.ActivityLogRepo != nil {
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package auth
|
||||
|
||||
import "strings"
|
||||
|
||||
// NormalizeUsername canonicalizes a username for storage and lookup by trimming
|
||||
// surrounding whitespace. Case is preserved for display; case-insensitive
|
||||
// matching is enforced by the citext column type in the database.
|
||||
func NormalizeUsername(username string) string {
|
||||
return strings.TrimSpace(username)
|
||||
}
|
||||
|
||||
// NormalizeEmail canonicalizes an email for storage and lookup by trimming
|
||||
// surrounding whitespace. Case is preserved (not lowercased); case-insensitive
|
||||
// matching is enforced by the citext column type in the database.
|
||||
func NormalizeEmail(email string) string {
|
||||
return strings.TrimSpace(email)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package auth
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeUsername(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"trims surrounding spaces", " john ", "john"},
|
||||
{"trims tabs and newlines", "\t john\n", "john"},
|
||||
{"preserves internal spacing", "john doe", "john doe"},
|
||||
{"preserves case", "JohnDoe", "JohnDoe"},
|
||||
{"whitespace only becomes empty", " ", ""},
|
||||
{"already clean is unchanged", "john", "john"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := NormalizeUsername(tc.in); got != tc.want {
|
||||
t.Errorf("NormalizeUsername(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeEmail(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"trims surrounding spaces", " user@example.com ", "user@example.com"},
|
||||
{"trims tabs and newlines", "\tuser@example.com\n", "user@example.com"},
|
||||
{"preserves case (not lowercased)", "User@Example.COM", "User@Example.COM"},
|
||||
{"whitespace only becomes empty", " ", ""},
|
||||
{"already clean is unchanged", "user@example.com", "user@example.com"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := NormalizeEmail(tc.in); got != tc.want {
|
||||
t.Errorf("NormalizeEmail(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -279,7 +279,7 @@ func (p *PluginProvider) autoProvisionUser(
|
||||
}
|
||||
|
||||
func randomPluginOnlyPassword() (string, error) {
|
||||
buf := make([]byte, 32)
|
||||
buf := make([]byte, 24)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
func TestRandomPluginOnlyPasswordFitsBcryptLimit(t *testing.T) {
|
||||
password, err := randomPluginOnlyPassword()
|
||||
if err != nil {
|
||||
t.Fatalf("randomPluginOnlyPassword() error = %v", err)
|
||||
}
|
||||
|
||||
if len(password) > 72 {
|
||||
t.Fatalf("password length = %d, want <= 72", len(password))
|
||||
}
|
||||
if _, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost); err != nil {
|
||||
t.Fatalf("bcrypt.GenerateFromPassword() error = %v", err)
|
||||
}
|
||||
}
|
||||
+20
-14
@@ -142,8 +142,8 @@ func (r *UserRepository) Create(ctx context.Context, input models.CreateUserInpu
|
||||
|
||||
cols := []string{"email", "username", "password_hash", "local_password_login_enabled", "role", "permissions", "library_ids", "max_playback_quality"}
|
||||
args := []any{
|
||||
input.Email,
|
||||
input.Username,
|
||||
NormalizeEmail(input.Email),
|
||||
NormalizeUsername(input.Username),
|
||||
string(hash),
|
||||
localPasswordLoginEnabled,
|
||||
input.Role,
|
||||
@@ -205,33 +205,34 @@ func (r *UserRepository) GetByID(ctx context.Context, id int) (*models.User, err
|
||||
return scanUser(r.pool.QueryRow(ctx, query, id))
|
||||
}
|
||||
|
||||
// GetByUsername retrieves a user by their username.
|
||||
// GetByUsername retrieves a user by their username (case-insensitive).
|
||||
func (r *UserRepository) GetByUsername(ctx context.Context, username string) (*models.User, error) {
|
||||
query := `SELECT ` + allColumns + ` FROM users WHERE username = $1`
|
||||
return scanUser(r.pool.QueryRow(ctx, query, username))
|
||||
return scanUser(r.pool.QueryRow(ctx, query, NormalizeUsername(username)))
|
||||
}
|
||||
|
||||
// GetByEmail retrieves a user by their email address.
|
||||
// GetByEmail retrieves a user by their email address (case-insensitive).
|
||||
func (r *UserRepository) GetByEmail(ctx context.Context, email string) (*models.User, error) {
|
||||
query := `SELECT ` + allColumns + ` FROM users WHERE email = $1`
|
||||
return scanUser(r.pool.QueryRow(ctx, query, email))
|
||||
return scanUser(r.pool.QueryRow(ctx, query, NormalizeEmail(email)))
|
||||
}
|
||||
|
||||
// Update modifies a user's fields. Only non-nil fields in the input are updated.
|
||||
// If the input contains a Password, it is bcrypt-hashed before storage.
|
||||
func (r *UserRepository) Update(ctx context.Context, id int, input models.UpdateUserInput) error {
|
||||
setClauses := []string{}
|
||||
accessPolicyPredicates := []string{}
|
||||
args := []any{}
|
||||
argIndex := 1
|
||||
|
||||
if input.Email != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("email = $%d", argIndex))
|
||||
args = append(args, *input.Email)
|
||||
args = append(args, NormalizeEmail(*input.Email))
|
||||
argIndex++
|
||||
}
|
||||
if input.Username != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("username = $%d", argIndex))
|
||||
args = append(args, *input.Username)
|
||||
args = append(args, NormalizeUsername(*input.Username))
|
||||
argIndex++
|
||||
}
|
||||
if input.Password != nil {
|
||||
@@ -250,6 +251,7 @@ func (r *UserRepository) Update(ctx context.Context, id int, input models.Update
|
||||
}
|
||||
if input.Role != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("role = $%d", argIndex))
|
||||
accessPolicyPredicates = append(accessPolicyPredicates, fmt.Sprintf("role IS DISTINCT FROM $%d", argIndex))
|
||||
args = append(args, *input.Role)
|
||||
argIndex++
|
||||
}
|
||||
@@ -259,21 +261,26 @@ func (r *UserRepository) Update(ctx context.Context, id int, input models.Update
|
||||
return err
|
||||
}
|
||||
setClauses = append(setClauses, fmt.Sprintf("permissions = $%d", argIndex))
|
||||
accessPolicyPredicates = append(accessPolicyPredicates, fmt.Sprintf("permissions IS DISTINCT FROM $%d", argIndex))
|
||||
args = append(args, permissions)
|
||||
argIndex++
|
||||
}
|
||||
if input.Enabled != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("enabled = $%d", argIndex))
|
||||
accessPolicyPredicates = append(accessPolicyPredicates, fmt.Sprintf("enabled IS DISTINCT FROM $%d", argIndex))
|
||||
args = append(args, *input.Enabled)
|
||||
argIndex++
|
||||
}
|
||||
if input.LibraryIDs != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("library_ids = $%d", argIndex))
|
||||
// Library scope is resolved from users.library_ids on each request, so
|
||||
// changing it must not invalidate durable profile/session tokens.
|
||||
args = append(args, *input.LibraryIDs)
|
||||
argIndex++
|
||||
}
|
||||
if input.MaxPlaybackQuality != nil {
|
||||
setClauses = append(setClauses, fmt.Sprintf("max_playback_quality = $%d", argIndex))
|
||||
accessPolicyPredicates = append(accessPolicyPredicates, fmt.Sprintf("max_playback_quality IS DISTINCT FROM $%d", argIndex))
|
||||
args = append(args, *input.MaxPlaybackQuality)
|
||||
argIndex++
|
||||
}
|
||||
@@ -309,12 +316,11 @@ func (r *UserRepository) Update(ctx context.Context, id int, input models.Update
|
||||
return err
|
||||
}
|
||||
|
||||
if input.Role != nil ||
|
||||
input.Enabled != nil ||
|
||||
input.LibraryIDs != nil ||
|
||||
input.MaxPlaybackQuality != nil ||
|
||||
input.Permissions != nil {
|
||||
setClauses = append(setClauses, "access_policy_revision = access_policy_revision + 1")
|
||||
if len(accessPolicyPredicates) > 0 {
|
||||
setClauses = append(setClauses, fmt.Sprintf(
|
||||
"access_policy_revision = CASE WHEN %s THEN access_policy_revision + 1 ELSE access_policy_revision END",
|
||||
strings.Join(accessPolicyPredicates, " OR "),
|
||||
))
|
||||
}
|
||||
|
||||
// Always bump updated_at.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ResolvedConnection is concrete credentials handed to the plugin.
|
||||
type ResolvedConnection struct {
|
||||
BaseURL string
|
||||
APIKey string
|
||||
}
|
||||
|
||||
// RequestIntegrationLookup resolves a soft-linked Requests integration to its
|
||||
// base URL and encrypted api-key reference.
|
||||
type RequestIntegrationLookup interface {
|
||||
Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error)
|
||||
}
|
||||
|
||||
// SecretResolver resolves an encrypted api-key reference to its plaintext value.
|
||||
type SecretResolver interface {
|
||||
Get(ctx context.Context, ref string) (string, error)
|
||||
}
|
||||
|
||||
type ConnectionResolver struct {
|
||||
requests RequestIntegrationLookup
|
||||
secrets SecretResolver
|
||||
}
|
||||
|
||||
func NewConnectionResolver(r RequestIntegrationLookup, s SecretResolver) *ConnectionResolver {
|
||||
return &ConnectionResolver{requests: r, secrets: s}
|
||||
}
|
||||
|
||||
// Resolve turns a stored Connection into concrete credentials. When the
|
||||
// connection is linked to a Requests integration the live base URL + key ref are
|
||||
// read from there; otherwise the connection's own fields are used. The api-key
|
||||
// ref is then resolved to plaintext via the secrets resolver.
|
||||
func (cr *ConnectionResolver) Resolve(ctx context.Context, c Connection) (ResolvedConnection, error) {
|
||||
baseURL, apiKeyRef := c.BaseURL, c.APIKeyRef
|
||||
// A pointer-to-empty-string request_integration_id is NOT a live link (it can
|
||||
// arise from a both-NULL orphan or a stripped link). Treat empty/whitespace as
|
||||
// "no link" so we don't call requests.Get("") and use the connection's own
|
||||
// fields instead.
|
||||
if c.RequestIntegrationID != nil && strings.TrimSpace(*c.RequestIntegrationID) != "" {
|
||||
if cr.requests == nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: no lookup configured", *c.RequestIntegrationID)
|
||||
}
|
||||
u, ref, err := cr.requests.Get(ctx, *c.RequestIntegrationID)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: %w", *c.RequestIntegrationID, err)
|
||||
}
|
||||
baseURL, apiKeyRef = u, ref
|
||||
}
|
||||
apiKey := strings.TrimSpace(apiKeyRef)
|
||||
if cr.secrets != nil && apiKey != "" {
|
||||
resolved, err := cr.secrets.Get(ctx, apiKey)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: resolve api key: %w", err)
|
||||
}
|
||||
if resolved = strings.TrimSpace(resolved); resolved != "" {
|
||||
apiKey = resolved
|
||||
}
|
||||
}
|
||||
return ResolvedConnection{BaseURL: baseURL, APIKey: apiKey}, nil
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeRequestLookup implements RequestIntegrationLookup from a static map.
|
||||
type fakeRequestLookup struct {
|
||||
entries map[string]struct{ baseURL, apiKeyRef string }
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeRequestLookup) Get(_ context.Context, id string) (string, string, error) {
|
||||
if f.err != nil {
|
||||
return "", "", f.err
|
||||
}
|
||||
e, ok := f.entries[id]
|
||||
if !ok {
|
||||
return "", "", errors.New("integration not found: " + id)
|
||||
}
|
||||
return e.baseURL, e.apiKeyRef, nil
|
||||
}
|
||||
|
||||
// fakeSecrets implements SecretResolver from a static map; unknown refs resolve
|
||||
// to empty (so the ref itself is used).
|
||||
type fakeSecrets struct {
|
||||
values map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeSecrets) Get(_ context.Context, ref string) (string, error) {
|
||||
if f.err != nil {
|
||||
return "", f.err
|
||||
}
|
||||
return f.values[ref], nil
|
||||
}
|
||||
|
||||
func TestResolveConnectionOwnCredentials(t *testing.T) {
|
||||
secrets := fakeSecrets{values: map[string]string{"ownref": "OWNKEY"}}
|
||||
r := NewConnectionResolver(fakeRequestLookup{}, secrets)
|
||||
|
||||
got, err := r.Resolve(context.Background(), Connection{
|
||||
BaseURL: "http://own:8989",
|
||||
APIKeyRef: "ownref",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve own: %v", err)
|
||||
}
|
||||
if got.BaseURL != "http://own:8989" || got.APIKey != "OWNKEY" {
|
||||
t.Fatalf("own resolve = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinked(t *testing.T) {
|
||||
lookup := fakeRequestLookup{entries: map[string]struct{ baseURL, apiKeyRef string }{
|
||||
"req-1": {baseURL: "http://req:7878", apiKeyRef: "rk"},
|
||||
}}
|
||||
secrets := fakeSecrets{values: map[string]string{"rk": "REQKEY"}}
|
||||
r := NewConnectionResolver(lookup, secrets)
|
||||
|
||||
id := "req-1"
|
||||
got, err := r.Resolve(context.Background(), Connection{RequestIntegrationID: &id})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve linked: %v", err)
|
||||
}
|
||||
if got.BaseURL != "http://req:7878" || got.APIKey != "REQKEY" {
|
||||
t.Fatalf("linked resolve = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinkedMissingErrors(t *testing.T) {
|
||||
r := NewConnectionResolver(fakeRequestLookup{}, fakeSecrets{})
|
||||
id := "gone"
|
||||
if _, err := r.Resolve(context.Background(), Connection{RequestIntegrationID: &id}); err == nil {
|
||||
t.Fatal("expected error when linked Requests integration is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinkedLookupErrors(t *testing.T) {
|
||||
r := NewConnectionResolver(fakeRequestLookup{err: errors.New("boom")}, fakeSecrets{})
|
||||
id := "req-1"
|
||||
if _, err := r.Resolve(context.Background(), Connection{RequestIntegrationID: &id}); err == nil {
|
||||
t.Fatal("expected error when the lookup itself fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionEmptyIntegrationIDIsNotALink(t *testing.T) {
|
||||
// A pointer-to-empty/whitespace request_integration_id must NOT be treated as
|
||||
// a live link: it must fall back to the connection's own fields, never call
|
||||
// requests.Get(""). The lookup here errors on any call, so reaching it fails.
|
||||
r := NewConnectionResolver(
|
||||
fakeRequestLookup{err: errors.New("lookup must not be called")},
|
||||
fakeSecrets{values: map[string]string{"ownref": "OWNKEY"}},
|
||||
)
|
||||
for _, empty := range []string{"", " "} {
|
||||
id := empty
|
||||
got, err := r.Resolve(context.Background(), Connection{
|
||||
BaseURL: "http://own:8989",
|
||||
APIKeyRef: "ownref",
|
||||
RequestIntegrationID: &id,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve with empty integration id %q: %v", empty, err)
|
||||
}
|
||||
if got.BaseURL != "http://own:8989" || got.APIKey != "OWNKEY" {
|
||||
t.Fatalf("empty integration id %q should use own creds, got %+v", empty, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionTrimsAndFallsBackOnEmptySecret(t *testing.T) {
|
||||
// Whitespace-padded ref; secret resolves to whitespace-only -> fall back to
|
||||
// the trimmed ref (parity with requests.resolveAPIKey).
|
||||
secrets := fakeSecrets{values: map[string]string{"ownref": " "}}
|
||||
r := NewConnectionResolver(fakeRequestLookup{}, secrets)
|
||||
|
||||
got, err := r.Resolve(context.Background(), Connection{
|
||||
BaseURL: "http://own:8989",
|
||||
APIKeyRef: " ownref ",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
}
|
||||
if got.APIKey != "ownref" {
|
||||
t.Fatalf("expected trimmed ref fallback, got %q", got.APIKey)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package autoscan
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
// uniqueParentDirs maps imported file paths to their distinct parent directories,
|
||||
// dropping empties. A season's episodes in one folder collapse to one entry.
|
||||
func uniqueParentDirs(paths []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var out []string
|
||||
for _, p := range paths {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(p)
|
||||
if _, ok := seen[dir]; ok {
|
||||
continue
|
||||
}
|
||||
seen[dir] = struct{}{}
|
||||
out = append(out, dir)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUniqueParentDirs(t *testing.T) {
|
||||
in := []string{
|
||||
"/mnt/media/Show/Season 01/E01.mkv",
|
||||
"/mnt/media/Show/Season 01/E02.mkv",
|
||||
"/mnt/media/Movie/Movie.mkv",
|
||||
"",
|
||||
}
|
||||
got := uniqueParentDirs(in)
|
||||
sort.Strings(got)
|
||||
want := []string{"/mnt/media/Movie", "/mnt/media/Show/Season 01"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("uniqueParentDirs = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// DiscoveredSource identifies one installed scan_source.v1 capability instance,
|
||||
// enriched with the metadata the Add-source picker needs (plugin id + a
|
||||
// human-friendly display name).
|
||||
type DiscoveredSource struct {
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
// PluginID is the installation's plugin id (e.g. "sonarr"); empty when the
|
||||
// lister cannot supply it.
|
||||
PluginID string
|
||||
// DisplayName is a human-friendly label for the capability (from the
|
||||
// capability's manifest display_name, falling back to plugin/capability ids).
|
||||
DisplayName string
|
||||
}
|
||||
|
||||
// ScanSourceLister enumerates every installed scan_source.v1 capability so the
|
||||
// engine can (a) offer them in the Add-source picker and (b) detect orphaned
|
||||
// source rows whose plugin has been uninstalled.
|
||||
type ScanSourceLister interface {
|
||||
// ListScanSources returns one entry per installed scan_source.v1 capability,
|
||||
// enriched with plugin id + display name.
|
||||
ListScanSources(ctx context.Context) ([]DiscoveredSource, error)
|
||||
}
|
||||
|
||||
// AvailableScanSource is one installed scan_source capability an operator can
|
||||
// create a source against (the Add-source picker list).
|
||||
type AvailableScanSource struct {
|
||||
InstallationID int `json:"installation_id"`
|
||||
CapabilityID string `json:"capability_id"`
|
||||
PluginID string `json:"plugin_id"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
// installedKey identifies an installed scan_source capability for set
|
||||
// membership tests (orphan detection in PollOnce).
|
||||
type installedKey struct {
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
}
|
||||
|
||||
// ListAvailableScanSources enumerates every installed scan_source capability so
|
||||
// an operator can pick one when creating a source. When no lister is configured
|
||||
// it returns an empty list. The handler also uses this to validate that a
|
||||
// create request targets a currently-installed capability.
|
||||
func (s *Service) ListAvailableScanSources(ctx context.Context) ([]AvailableScanSource, error) {
|
||||
if s.lister == nil {
|
||||
return []AvailableScanSource{}, nil
|
||||
}
|
||||
discovered, err := s.lister.ListScanSources(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list scan sources: %w", err)
|
||||
}
|
||||
out := make([]AvailableScanSource, 0, len(discovered))
|
||||
for _, d := range discovered {
|
||||
out = append(out, AvailableScanSource{
|
||||
InstallationID: d.InstallationID,
|
||||
CapabilityID: d.CapabilityID,
|
||||
PluginID: d.PluginID,
|
||||
DisplayName: d.DisplayName,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// installedScanSources returns the set of currently-installed scan_source
|
||||
// capabilities so PollOnce can skip orphaned source rows (their plugin is
|
||||
// gone). A nil set means the set is unavailable (no lister) — callers must treat
|
||||
// that as "discovery unavailable" and NOT prune.
|
||||
func (s *Service) installedScanSources(ctx context.Context) (map[installedKey]struct{}, error) {
|
||||
if s.lister == nil {
|
||||
return nil, nil
|
||||
}
|
||||
discovered, err := s.lister.ListScanSources(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list scan sources: %w", err)
|
||||
}
|
||||
present := make(map[installedKey]struct{}, len(discovered))
|
||||
for _, d := range discovered {
|
||||
present[installedKey{InstallationID: d.InstallationID, CapabilityID: d.CapabilityID}] = struct{}{}
|
||||
}
|
||||
return present, nil
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeLister returns a fixed set of discovered scan sources.
|
||||
type fakeLister struct{ sources []DiscoveredSource }
|
||||
|
||||
func (f fakeLister) ListScanSources(context.Context) ([]DiscoveredSource, error) {
|
||||
return f.sources, nil
|
||||
}
|
||||
|
||||
func TestListAvailableScanSourcesEnumeratesInstalled(t *testing.T) {
|
||||
lister := fakeLister{sources: []DiscoveredSource{
|
||||
{InstallationID: 1, CapabilityID: "arr-a", PluginID: "sonarr", DisplayName: "Sonarr"},
|
||||
{InstallationID: 2, CapabilityID: "arr-b", PluginID: "radarr", DisplayName: "Radarr"},
|
||||
}}
|
||||
svc := &Service{lister: lister}
|
||||
|
||||
available, err := svc.ListAvailableScanSources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListAvailableScanSources: %v", err)
|
||||
}
|
||||
if len(available) != 2 {
|
||||
t.Fatalf("expected 2 available, got %d: %+v", len(available), available)
|
||||
}
|
||||
if available[0] != (AvailableScanSource{InstallationID: 1, CapabilityID: "arr-a", PluginID: "sonarr", DisplayName: "Sonarr"}) {
|
||||
t.Fatalf("unexpected first available: %+v", available[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAvailableScanSourcesNilListerEmpty(t *testing.T) {
|
||||
svc := &Service{lister: nil}
|
||||
available, err := svc.ListAvailableScanSources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("ListAvailableScanSources: %v", err)
|
||||
}
|
||||
if len(available) != 0 {
|
||||
t.Fatalf("nil lister must return empty, got %+v", available)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledScanSourcesSetMembership(t *testing.T) {
|
||||
lister := fakeLister{sources: []DiscoveredSource{
|
||||
{InstallationID: 1, CapabilityID: "arr-a"},
|
||||
}}
|
||||
svc := &Service{lister: lister}
|
||||
|
||||
present, err := svc.installedScanSources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("installedScanSources: %v", err)
|
||||
}
|
||||
if _, ok := present[installedKey{1, "arr-a"}]; !ok {
|
||||
t.Fatalf("expected 1/arr-a present: %+v", present)
|
||||
}
|
||||
if _, ok := present[installedKey{2, "arr-b"}]; ok {
|
||||
t.Fatalf("did not expect 2/arr-b present: %+v", present)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstalledScanSourcesNilListerNilSet(t *testing.T) {
|
||||
svc := &Service{lister: nil}
|
||||
present, err := svc.installedScanSources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("installedScanSources: %v", err)
|
||||
}
|
||||
if present != nil {
|
||||
t.Fatalf("nil lister must return a nil set, got %+v", present)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package autoscan
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrNotFound is returned when an autoscan operation references a connection or
|
||||
// source that does not exist. The admin API surfaces it as a 404.
|
||||
var ErrNotFound = errors.New("autoscan: not found")
|
||||
@@ -0,0 +1,20 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// MaxSourceLabelLen bounds an operator-set source label in runes.
|
||||
const MaxSourceLabelLen = 120
|
||||
|
||||
// NormalizeSourceLabel trims surrounding whitespace and caps the label at
|
||||
// MaxSourceLabelLen runes without splitting a multi-byte rune. An all-whitespace
|
||||
// label normalizes to "" (unset).
|
||||
func NormalizeSourceLabel(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if utf8.RuneCountInString(s) <= MaxSourceLabelLen {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:MaxSourceLabelLen])
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestNormalizeSourceLabel(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"trims surrounding whitespace", " 4K Movies ", "4K Movies"},
|
||||
{"empty stays empty", "", ""},
|
||||
{"all whitespace becomes empty", " ", ""},
|
||||
{"under cap unchanged", strings.Repeat("x", 120), strings.Repeat("x", 120)},
|
||||
{"over cap truncated to 120 runes", strings.Repeat("x", 200), strings.Repeat("x", 120)},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := NormalizeSourceLabel(c.in); got != c.want {
|
||||
t.Fatalf("NormalizeSourceLabel(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSourceLabelRuneSafe(t *testing.T) {
|
||||
// 200 multi-byte runes must cap to 120 runes and stay valid UTF-8.
|
||||
got := NormalizeSourceLabel(strings.Repeat("é", 200))
|
||||
if utf8.RuneCountInString(got) != 120 {
|
||||
t.Fatalf("rune count = %d, want 120", utf8.RuneCountInString(got))
|
||||
}
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatal("result is not valid UTF-8")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ArrStatusProbe probes an arr instance's /api/v3/system/status to confirm the
|
||||
// base URL + api key are reachable and valid. It returns the reported version
|
||||
// on success.
|
||||
type ArrStatusProbe interface {
|
||||
SystemStatus(ctx context.Context, baseURL, apiKey string) (version string, err error)
|
||||
}
|
||||
|
||||
// ConnectionTestResult is the outcome of probing a connection's arr endpoint.
|
||||
type ConnectionTestResult struct {
|
||||
OK bool
|
||||
Version string
|
||||
Err string
|
||||
}
|
||||
|
||||
// TestConnection resolves the given connection to concrete credentials and
|
||||
// probes its arr /api/v3/system/status. A resolve failure or an unreachable /
|
||||
// unauthorized / non-200 endpoint yields OK=false with a human-readable Err
|
||||
// (never an HTTP error from this method itself — probe failures are part of the
|
||||
// successful response payload). A nil probe dependency is a wiring error and is
|
||||
// returned as a real error.
|
||||
func (s *Service) TestConnection(ctx context.Context, c Connection) (ConnectionTestResult, error) {
|
||||
if s.rootFolders == nil {
|
||||
return ConnectionTestResult{}, fmt.Errorf("autoscan: connection probe not configured")
|
||||
}
|
||||
probe, ok := s.rootFolders.(ArrStatusProbe)
|
||||
if !ok {
|
||||
return ConnectionTestResult{}, fmt.Errorf("autoscan: connection probe not configured")
|
||||
}
|
||||
resolved, err := s.connres.Resolve(ctx, c)
|
||||
if err != nil {
|
||||
return ConnectionTestResult{OK: false, Err: err.Error()}, nil
|
||||
}
|
||||
version, err := probe.SystemStatus(ctx, resolved.BaseURL, resolved.APIKey)
|
||||
if err != nil {
|
||||
return ConnectionTestResult{OK: false, Err: err.Error()}, nil
|
||||
}
|
||||
return ConnectionTestResult{OK: true, Version: version}, nil
|
||||
}
|
||||
|
||||
// TestConnectionByID loads a stored connection by id and tests it.
|
||||
func (s *Service) TestConnectionByID(ctx context.Context, id string) (ConnectionTestResult, error) {
|
||||
c, err := s.store.GetConnection(ctx, id)
|
||||
if err != nil {
|
||||
return ConnectionTestResult{}, err
|
||||
}
|
||||
return s.TestConnection(ctx, c)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeArrClient implements both RootFolderClient and ArrStatusProbe for the
|
||||
// connection-test and suggester service tests.
|
||||
type fakeArrClient struct {
|
||||
roots []string
|
||||
rootsErr error
|
||||
version string
|
||||
statusErr error
|
||||
gotBaseURL string
|
||||
gotAPIKey string
|
||||
}
|
||||
|
||||
func (f *fakeArrClient) RootFolders(_ context.Context, baseURL, apiKey string) ([]string, error) {
|
||||
f.gotBaseURL, f.gotAPIKey = baseURL, apiKey
|
||||
return f.roots, f.rootsErr
|
||||
}
|
||||
|
||||
func (f *fakeArrClient) SystemStatus(_ context.Context, baseURL, apiKey string) (string, error) {
|
||||
f.gotBaseURL, f.gotAPIKey = baseURL, apiKey
|
||||
return f.version, f.statusErr
|
||||
}
|
||||
|
||||
type fakeFolderLister struct{ paths []string }
|
||||
|
||||
func (f fakeFolderLister) ListFolderPaths(context.Context) ([]string, error) {
|
||||
return f.paths, nil
|
||||
}
|
||||
|
||||
func TestTestConnectionOK(t *testing.T) {
|
||||
arr := &fakeArrClient{version: "4.1.2.3"}
|
||||
svc := &Service{connres: passthroughConnRes{}, rootFolders: arr}
|
||||
|
||||
res, err := svc.TestConnection(context.Background(), Connection{BaseURL: "http://radarr:7878", APIKeyRef: "k"})
|
||||
if err != nil {
|
||||
t.Fatalf("TestConnection: %v", err)
|
||||
}
|
||||
if !res.OK || res.Version != "4.1.2.3" {
|
||||
t.Fatalf("expected ok with version, got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestConnectionProbeFailureIsPayload(t *testing.T) {
|
||||
arr := &fakeArrClient{statusErr: errors.New("arr: HTTP 401")}
|
||||
svc := &Service{connres: passthroughConnRes{}, rootFolders: arr}
|
||||
|
||||
res, err := svc.TestConnection(context.Background(), Connection{BaseURL: "http://radarr:7878"})
|
||||
if err != nil {
|
||||
t.Fatalf("probe failure must not be a method error: %v", err)
|
||||
}
|
||||
if res.OK || res.Err == "" {
|
||||
t.Fatalf("expected ok=false with error, got %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestConnectionNoProbeConfigured(t *testing.T) {
|
||||
svc := &Service{connres: passthroughConnRes{}}
|
||||
if _, err := svc.TestConnection(context.Background(), Connection{BaseURL: "http://x"}); err == nil {
|
||||
t.Fatal("expected error when no probe configured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestRewritesMatchesArrRootsToSiloFolders(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
sources: []Source{
|
||||
{ID: "src-1", InstallationID: 1, CapabilityID: "arr", ConnectionID: strptr("c1")},
|
||||
},
|
||||
}
|
||||
arr := &fakeArrClient{roots: []string{"/data/tv"}}
|
||||
folders := fakeFolderLister{paths: []string{"/mnt/media/tv"}}
|
||||
svc := &Service{store: store, connres: passthroughConnRes{}, rootFolders: arr, folders: folders}
|
||||
|
||||
sugg, err := svc.SuggestRewrites(context.Background(), "src-1")
|
||||
if err != nil {
|
||||
t.Fatalf("SuggestRewrites: %v", err)
|
||||
}
|
||||
if len(sugg.Proposed) != 1 || sugg.Proposed[0].From != "/data/tv" || sugg.Proposed[0].To != "/mnt/media/tv" {
|
||||
t.Fatalf("unexpected proposals: %+v", sugg.Proposed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuggestRewritesNoConnection(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
sources: []Source{{ID: "src-1", InstallationID: 1, CapabilityID: "arr", ConnectionID: nil}},
|
||||
}
|
||||
svc := &Service{store: store, connres: passthroughConnRes{}, rootFolders: &fakeArrClient{}, folders: fakeFolderLister{}}
|
||||
|
||||
if _, err := svc.SuggestRewrites(context.Background(), "src-1"); !errors.Is(err, ErrNoConnection) {
|
||||
t.Fatalf("expected ErrNoConnection, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
)
|
||||
|
||||
// ScanSourceProvider yields changed paths for one source. The engine calls
|
||||
// PollChanges; production wraps the plugins.Service scan_source resolver.
|
||||
type ScanSourceProvider interface {
|
||||
PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection, sourceConfig map[string]string) (changes []Change, nextMarker string, err error)
|
||||
}
|
||||
|
||||
// PollChangesClient is the slice of *pluginhost.ScanSourceClient used here. It
|
||||
// is exported so wiring in the api package can declare an adapter whose
|
||||
// ScanSourceClient returns it (Go interface method signatures must match
|
||||
// exactly across packages, and the unexported form could not be named there).
|
||||
type PollChangesClient interface {
|
||||
PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error)
|
||||
}
|
||||
|
||||
// ScanSourceResolver yields a per-(installation, capability) scan-source client.
|
||||
// Exported for the same cross-package adapter reason as PollChangesClient.
|
||||
type ScanSourceResolver interface {
|
||||
ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (PollChangesClient, error)
|
||||
}
|
||||
|
||||
type pluginProvider struct{ resolver ScanSourceResolver }
|
||||
|
||||
// NewPluginProvider builds the production scan-source provider over the plugins
|
||||
// resolver.
|
||||
func NewPluginProvider(resolver ScanSourceResolver) ScanSourceProvider {
|
||||
return &pluginProvider{resolver: resolver}
|
||||
}
|
||||
|
||||
func (p *pluginProvider) PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection, sourceConfig map[string]string) ([]Change, string, error) {
|
||||
client, err := p.resolver.ScanSourceClient(ctx, installationID, capabilityID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
resp, err := client.PollChanges(ctx, &pluginv1.PollChangesRequest{
|
||||
CapabilityId: capabilityID,
|
||||
Marker: marker,
|
||||
Connection: &pluginv1.ResolvedConnection{BaseUrl: conn.BaseURL, ApiKey: conn.APIKey},
|
||||
SourceConfig: sourceConfig,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if structured := resp.GetChanges(); len(structured) > 0 {
|
||||
changes := make([]Change, 0, len(structured))
|
||||
for _, change := range structured {
|
||||
if change == nil {
|
||||
continue
|
||||
}
|
||||
changes = append(changes, Change{
|
||||
SourcePath: change.GetSourcePath(),
|
||||
Scope: scanSourceScope(change.GetScope()),
|
||||
})
|
||||
}
|
||||
return changes, resp.GetNextMarker(), nil
|
||||
}
|
||||
|
||||
// Legacy plugins return only source_paths. Treat them as auto/file-like
|
||||
// paths so the existing parent-directory collapse behavior is preserved.
|
||||
changes := make([]Change, 0, len(resp.GetSourcePaths()))
|
||||
for _, path := range resp.GetSourcePaths() {
|
||||
changes = append(changes, Change{SourcePath: path, Scope: ChangeScopeAuto})
|
||||
}
|
||||
return changes, resp.GetNextMarker(), nil
|
||||
}
|
||||
|
||||
func scanSourceScope(scope pluginv1.ScanSourceChangeScope) ChangeScope {
|
||||
switch scope {
|
||||
case pluginv1.ScanSourceChangeScope_SCAN_SOURCE_CHANGE_SCOPE_FILE:
|
||||
return ChangeScopeFile
|
||||
case pluginv1.ScanSourceChangeScope_SCAN_SOURCE_CHANGE_SCOPE_SUBTREE:
|
||||
return ChangeScopeSubtree
|
||||
default:
|
||||
return ChangeScopeAuto
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
)
|
||||
|
||||
// capturingClient records the last PollChangesRequest it was handed.
|
||||
type capturingClient struct {
|
||||
last *pluginv1.PollChangesRequest
|
||||
}
|
||||
|
||||
func (c *capturingClient) PollChanges(_ context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
c.last = req
|
||||
return &pluginv1.PollChangesResponse{SourcePaths: []string{"/mnt/media/x"}, NextMarker: "m2"}, nil
|
||||
}
|
||||
|
||||
// capturingResolver yields a fixed PollChangesClient.
|
||||
type capturingResolver struct{ client PollChangesClient }
|
||||
|
||||
func (r capturingResolver) ScanSourceClient(context.Context, int, string) (PollChangesClient, error) {
|
||||
return r.client, nil
|
||||
}
|
||||
|
||||
// TestPluginProviderPopulatesConnection asserts the resolved connection is
|
||||
// delivered to the plugin on the PollChangesRequest.
|
||||
func TestPluginProviderPopulatesConnection(t *testing.T) {
|
||||
client := &capturingClient{}
|
||||
prov := NewPluginProvider(capturingResolver{client})
|
||||
|
||||
conn := ResolvedConnection{BaseURL: "https://arr.example", APIKey: "secret-key"}
|
||||
sourceConfig := map[string]string{"exclusions": ".downloads"}
|
||||
changes, next, err := prov.PollChanges(context.Background(), 1, "cap", "m1", conn, sourceConfig)
|
||||
if err != nil {
|
||||
t.Fatalf("PollChanges: %v", err)
|
||||
}
|
||||
if len(changes) != 1 || changes[0].SourcePath != "/mnt/media/x" || changes[0].Scope != ChangeScopeAuto || next != "m2" {
|
||||
t.Fatalf("unexpected provider result: changes=%v next=%q", changes, next)
|
||||
}
|
||||
if client.last == nil {
|
||||
t.Fatal("expected a PollChangesRequest to be sent")
|
||||
}
|
||||
if client.last.GetCapabilityId() != "cap" || client.last.GetMarker() != "m1" {
|
||||
t.Fatalf("unexpected request fields: cap=%q marker=%q", client.last.GetCapabilityId(), client.last.GetMarker())
|
||||
}
|
||||
rc := client.last.GetConnection()
|
||||
if rc == nil {
|
||||
t.Fatal("expected connection to be populated on the request")
|
||||
}
|
||||
if rc.GetBaseUrl() != conn.BaseURL || rc.GetApiKey() != conn.APIKey {
|
||||
t.Fatalf("connection not delivered: base_url=%q api_key=%q", rc.GetBaseUrl(), rc.GetApiKey())
|
||||
}
|
||||
if client.last.GetSourceConfig()["exclusions"] != ".downloads" {
|
||||
t.Fatalf("source_config not delivered: %#v", client.last.GetSourceConfig())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginProviderPrefersStructuredChanges(t *testing.T) {
|
||||
client := &capturingStructuredClient{}
|
||||
prov := NewPluginProvider(capturingResolver{client: client})
|
||||
|
||||
changes, next, err := prov.PollChanges(context.Background(), 1, "cap", "m1", ResolvedConnection{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("PollChanges: %v", err)
|
||||
}
|
||||
if next != "m3" || len(changes) != 2 {
|
||||
t.Fatalf("unexpected provider result: changes=%v next=%q", changes, next)
|
||||
}
|
||||
if changes[0] != (Change{SourcePath: "/ceph/movie/Movie", Scope: ChangeScopeSubtree}) {
|
||||
t.Fatalf("first change = %#v", changes[0])
|
||||
}
|
||||
if changes[1] != (Change{SourcePath: "/ceph/show/S01/E01.mkv", Scope: ChangeScopeFile}) {
|
||||
t.Fatalf("second change = %#v", changes[1])
|
||||
}
|
||||
}
|
||||
|
||||
type capturingStructuredClient struct {
|
||||
last *pluginv1.PollChangesRequest
|
||||
}
|
||||
|
||||
func (c *capturingStructuredClient) PollChanges(_ context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
c.last = req
|
||||
return &pluginv1.PollChangesResponse{
|
||||
SourcePaths: []string{"/legacy/ignored.mkv"},
|
||||
NextMarker: "m3",
|
||||
Changes: []*pluginv1.ScanSourceChange{
|
||||
{
|
||||
SourcePath: "/ceph/movie/Movie",
|
||||
Scope: pluginv1.ScanSourceChangeScope_SCAN_SOURCE_CHANGE_SCOPE_SUBTREE,
|
||||
},
|
||||
{
|
||||
SourcePath: "/ceph/show/S01/E01.mkv",
|
||||
Scope: pluginv1.ScanSourceChangeScope_SCAN_SOURCE_CHANGE_SCOPE_FILE,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,861 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Repository struct{ pool *pgxpool.Pool }
|
||||
|
||||
func NewRepository(pool *pgxpool.Pool) *Repository { return &Repository{pool: pool} }
|
||||
|
||||
// --- Settings ---
|
||||
|
||||
func (r *Repository) GetSettings(ctx context.Context) (Settings, error) {
|
||||
var s Settings
|
||||
err := r.pool.QueryRow(ctx, `
|
||||
SELECT enabled, default_poll_interval_seconds, debounce_seconds
|
||||
FROM autoscan_settings WHERE id = true`).
|
||||
Scan(&s.Enabled, &s.DefaultPollIntervalSeconds, &s.DebounceSeconds)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Settings{
|
||||
Enabled: false,
|
||||
DefaultPollIntervalSeconds: 600,
|
||||
DebounceSeconds: 60,
|
||||
}, nil
|
||||
}
|
||||
return Settings{}, fmt.Errorf("get autoscan settings: %w", err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (r *Repository) UpdateSettings(ctx context.Context, s Settings) (Settings, error) {
|
||||
var out Settings
|
||||
err := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_settings (
|
||||
id, enabled, default_poll_interval_seconds, debounce_seconds, updated_at
|
||||
)
|
||||
VALUES (true, $1, $2, $3, now())
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
enabled = EXCLUDED.enabled,
|
||||
default_poll_interval_seconds = EXCLUDED.default_poll_interval_seconds,
|
||||
debounce_seconds = EXCLUDED.debounce_seconds,
|
||||
updated_at = now()
|
||||
RETURNING enabled, default_poll_interval_seconds, debounce_seconds`,
|
||||
s.Enabled, s.DefaultPollIntervalSeconds, s.DebounceSeconds).
|
||||
Scan(&out.Enabled, &out.DefaultPollIntervalSeconds, &out.DebounceSeconds)
|
||||
if err != nil {
|
||||
return Settings{}, fmt.Errorf("update autoscan settings: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// --- Connections ---
|
||||
|
||||
const connectionColumns = `id, name, kind, base_url, api_key_ref, request_integration_id`
|
||||
|
||||
func scanConnection(row interface{ Scan(...any) error }) (Connection, error) {
|
||||
var c Connection
|
||||
var baseURL, apiKeyRef, reqIntegrationID *string
|
||||
if err := row.Scan(&c.ID, &c.Name, &c.Kind, &baseURL, &apiKeyRef, &reqIntegrationID); err != nil {
|
||||
return Connection{}, err
|
||||
}
|
||||
if baseURL != nil {
|
||||
c.BaseURL = *baseURL
|
||||
}
|
||||
if apiKeyRef != nil {
|
||||
c.APIKeyRef = *apiKeyRef
|
||||
}
|
||||
c.RequestIntegrationID = reqIntegrationID
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// nullable returns nil for empty strings so they map to SQL NULL.
|
||||
func nullable(s string) any {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (r *Repository) CreateConnection(ctx context.Context, c Connection) (Connection, error) {
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_connections (name, kind, base_url, api_key_ref, request_integration_id)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING `+connectionColumns,
|
||||
c.Name, c.Kind, nullable(c.BaseURL), nullable(c.APIKeyRef), c.RequestIntegrationID)
|
||||
out, err := scanConnection(row)
|
||||
if err != nil {
|
||||
return Connection{}, fmt.Errorf("create autoscan connection: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Repository) UpdateConnection(ctx context.Context, c Connection) (Connection, error) {
|
||||
// A blank incoming api_key_ref KEEPS the existing stored value: the UI
|
||||
// deliberately omits the key on a metadata-only edit ("leave blank to keep
|
||||
// existing"), so unconditionally writing it would NULL the key and break the
|
||||
// next poll. Mirrors requests.UpdateIntegration's CASE-WHEN keep-semantics.
|
||||
// Pass the raw trimmed string (not nullable()) so the empty-string sentinel
|
||||
// reaches the CASE.
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
UPDATE autoscan_connections
|
||||
SET name = $2, kind = $3, base_url = $4,
|
||||
api_key_ref = CASE WHEN $5 = '' THEN api_key_ref ELSE $5 END,
|
||||
request_integration_id = $6, updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING `+connectionColumns,
|
||||
c.ID, c.Name, c.Kind, nullable(c.BaseURL), c.APIKeyRef, c.RequestIntegrationID)
|
||||
out, err := scanConnection(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Connection{}, fmt.Errorf("%w: connection %s", ErrNotFound, c.ID)
|
||||
}
|
||||
return Connection{}, fmt.Errorf("update autoscan connection: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Repository) DeleteConnection(ctx context.Context, id string) error {
|
||||
tag, err := r.pool.Exec(ctx, `DELETE FROM autoscan_connections WHERE id = $1`, id)
|
||||
if err != nil {
|
||||
// A source still references this connection (ON DELETE RESTRICT, 23503).
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23503" {
|
||||
return fmt.Errorf("autoscan: connection %s is in use by a source", id)
|
||||
}
|
||||
return fmt.Errorf("delete autoscan connection: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: connection %s", ErrNotFound, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Repository) ListConnections(ctx context.Context) ([]Connection, error) {
|
||||
rows, err := r.pool.Query(ctx, `SELECT `+connectionColumns+`
|
||||
FROM autoscan_connections ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list autoscan connections: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Connection
|
||||
for rows.Next() {
|
||||
c, err := scanConnection(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Repository) GetConnection(ctx context.Context, id string) (Connection, error) {
|
||||
row := r.pool.QueryRow(ctx, `SELECT `+connectionColumns+`
|
||||
FROM autoscan_connections WHERE id = $1`, id)
|
||||
c, err := scanConnection(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Connection{}, fmt.Errorf("%w: connection %s", ErrNotFound, id)
|
||||
}
|
||||
return Connection{}, fmt.Errorf("get autoscan connection: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// --- Sources ---
|
||||
|
||||
const sourceColumns = `id, installation_id, capability_id, connection_id, enabled,
|
||||
poll_interval_seconds, path_rewrites, source_config, label, marker, last_run_at, last_error`
|
||||
|
||||
func scanSource(row interface{ Scan(...any) error }) (Source, error) {
|
||||
var s Source
|
||||
var pathRewrites []byte
|
||||
var sourceConfig []byte
|
||||
if err := row.Scan(&s.ID, &s.InstallationID, &s.CapabilityID, &s.ConnectionID,
|
||||
&s.Enabled, &s.PollIntervalSeconds, &pathRewrites, &sourceConfig, &s.Label, &s.Marker, &s.LastRunAt, &s.LastError); err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
rewrites, err := unmarshalPathRewrites(pathRewrites)
|
||||
if err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
s.PathRewrites = rewrites
|
||||
config, err := unmarshalSourceConfig(sourceConfig)
|
||||
if err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
s.SourceConfig = config
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// unmarshalPathRewrites decodes the jsonb path_rewrites column into a slice. A
|
||||
// NULL/empty column maps to an empty (non-nil) slice so callers never see a nil.
|
||||
func unmarshalPathRewrites(raw []byte) ([]PathRewrite, error) {
|
||||
if len(raw) == 0 {
|
||||
return []PathRewrite{}, nil
|
||||
}
|
||||
var out []PathRewrite
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode autoscan path_rewrites: %w", err)
|
||||
}
|
||||
if out == nil {
|
||||
out = []PathRewrite{}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// marshalPathRewrites encodes path rewrites for the jsonb column. A nil slice is
|
||||
// stored as an empty JSON array (matching the column default '[]').
|
||||
func marshalPathRewrites(rewrites []PathRewrite) ([]byte, error) {
|
||||
if rewrites == nil {
|
||||
rewrites = []PathRewrite{}
|
||||
}
|
||||
b, err := json.Marshal(rewrites)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode autoscan path_rewrites: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func unmarshalSourceConfig(raw []byte) (map[string]string, error) {
|
||||
if len(raw) == 0 {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
var out map[string]string
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode autoscan source_config: %w", err)
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func marshalSourceConfig(config map[string]string) ([]byte, error) {
|
||||
normalized := make(map[string]string, len(config))
|
||||
for key, value := range config {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
normalized[key] = strings.TrimSpace(value)
|
||||
}
|
||||
b, err := json.Marshal(normalized)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode autoscan source_config: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// connectionIDArg maps a nullable connection id to a SQL value: nil pointer and
|
||||
// whitespace-only ids map to SQL NULL.
|
||||
func connectionIDArg(id *string) any {
|
||||
if id == nil {
|
||||
return nil
|
||||
}
|
||||
return nullable(*id)
|
||||
}
|
||||
|
||||
// CreateSource inserts a new autoscan source row. One installed scan_source
|
||||
// capability can back many sources (e.g. one Sonarr plugin install fronting 4
|
||||
// arr servers, one source per connection), so this is a plain INSERT with a
|
||||
// fresh uuid rather than an upsert. A non-existent connection trips the FK
|
||||
// constraint and maps to ErrNotFound.
|
||||
func (r *Repository) CreateSource(ctx context.Context, s Source) (Source, error) {
|
||||
rewrites, err := marshalPathRewrites(s.PathRewrites)
|
||||
if err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
sourceConfig, err := marshalSourceConfig(s.SourceConfig)
|
||||
if err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_sources (
|
||||
installation_id, capability_id, connection_id, enabled, poll_interval_seconds, path_rewrites, source_config, label
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING `+sourceColumns,
|
||||
s.InstallationID, s.CapabilityID, connectionIDArg(s.ConnectionID), s.Enabled, s.PollIntervalSeconds, rewrites, sourceConfig, s.Label)
|
||||
out, err := scanSource(row)
|
||||
if err != nil {
|
||||
if connID, ok := connectionFKViolation(err, s.ConnectionID); ok {
|
||||
return Source{}, fmt.Errorf("%w: connection %s", ErrNotFound, connID)
|
||||
}
|
||||
return Source{}, fmt.Errorf("create autoscan source: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// UpdateSource updates a source's binding/scheduling fields by id. Identity
|
||||
// (installation_id, capability_id) and bookkeeping fields (marker/last_run_at/
|
||||
// last_error) are left untouched. An unknown id maps to ErrNotFound; a
|
||||
// non-existent connection trips the FK constraint and also maps to ErrNotFound.
|
||||
func (r *Repository) UpdateSource(ctx context.Context, s Source) (Source, error) {
|
||||
rewrites, err := marshalPathRewrites(s.PathRewrites)
|
||||
if err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
sourceConfig, err := marshalSourceConfig(s.SourceConfig)
|
||||
if err != nil {
|
||||
return Source{}, err
|
||||
}
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
UPDATE autoscan_sources
|
||||
SET connection_id = $2,
|
||||
enabled = $3,
|
||||
poll_interval_seconds = $4,
|
||||
path_rewrites = $5,
|
||||
source_config = $6,
|
||||
label = $7,
|
||||
updated_at = now()
|
||||
WHERE id = $1
|
||||
RETURNING `+sourceColumns,
|
||||
s.ID, connectionIDArg(s.ConnectionID), s.Enabled, s.PollIntervalSeconds, rewrites, sourceConfig, s.Label)
|
||||
out, err := scanSource(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Source{}, fmt.Errorf("%w: source %s", ErrNotFound, s.ID)
|
||||
}
|
||||
if connID, ok := connectionFKViolation(err, s.ConnectionID); ok {
|
||||
return Source{}, fmt.Errorf("%w: connection %s", ErrNotFound, connID)
|
||||
}
|
||||
return Source{}, fmt.Errorf("update autoscan source: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// connectionFKViolation reports whether err is the connection FK constraint
|
||||
// violation (23503), returning the offending connection id for the error
|
||||
// message.
|
||||
func connectionFKViolation(err error, connectionID *string) (string, bool) {
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) || pgErr.Code != "23503" {
|
||||
return "", false
|
||||
}
|
||||
connID := ""
|
||||
if connectionID != nil {
|
||||
connID = *connectionID
|
||||
}
|
||||
return connID, true
|
||||
}
|
||||
|
||||
func (r *Repository) ListSources(ctx context.Context) ([]Source, error) {
|
||||
rows, err := r.pool.Query(ctx, `SELECT `+sourceColumns+`
|
||||
FROM autoscan_sources ORDER BY installation_id, capability_id`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list autoscan sources: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
return collectSources(rows)
|
||||
}
|
||||
|
||||
func (r *Repository) ListEnabledSources(ctx context.Context) ([]Source, error) {
|
||||
rows, err := r.pool.Query(ctx, `SELECT `+sourceColumns+`
|
||||
FROM autoscan_sources WHERE enabled = true
|
||||
ORDER BY installation_id, capability_id`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list enabled autoscan sources: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
return collectSources(rows)
|
||||
}
|
||||
|
||||
func collectSources(rows pgx.Rows) ([]Source, error) {
|
||||
var out []Source
|
||||
for rows.Next() {
|
||||
s, err := scanSource(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Repository) GetSource(ctx context.Context, id string) (Source, error) {
|
||||
row := r.pool.QueryRow(ctx, `SELECT `+sourceColumns+`
|
||||
FROM autoscan_sources WHERE id = $1`, id)
|
||||
s, err := scanSource(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Source{}, fmt.Errorf("%w: source %s", ErrNotFound, id)
|
||||
}
|
||||
return Source{}, fmt.Errorf("get autoscan source: %w", err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// DeleteSource removes a source row by id. It lets an operator clear an orphaned
|
||||
// source (one whose scan_source plugin was uninstalled/disabled). An unknown id
|
||||
// maps to ErrNotFound.
|
||||
func (r *Repository) DeleteSource(ctx context.Context, id string) error {
|
||||
tag, err := r.pool.Exec(ctx, `DELETE FROM autoscan_sources WHERE id = $1`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete autoscan source: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: source %s", ErrNotFound, id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AdvanceMarker stores the opaque next marker for a source, stamps last_run_at,
|
||||
// and clears any prior error. Called only after a successful enqueue.
|
||||
func (r *Repository) AdvanceMarker(ctx context.Context, sourceID, marker string) error {
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE autoscan_sources
|
||||
SET marker = $2, last_run_at = now(), last_error = NULL, updated_at = now()
|
||||
WHERE id = $1`, sourceID, nullable(marker))
|
||||
if err != nil {
|
||||
return fmt.Errorf("advance autoscan marker: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: source %s", ErrNotFound, sourceID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// maxLastErrorLen bounds the stored last_error (in bytes) so a pathological
|
||||
// provider error can't bloat the row.
|
||||
const maxLastErrorLen = 2048
|
||||
|
||||
// truncateUTF8 caps s to at most maxBytes bytes without splitting a multi-byte
|
||||
// UTF-8 rune: if the byte cut lands mid-rune it backs off to the last valid
|
||||
// rune boundary, so the stored value is always valid UTF-8.
|
||||
func truncateUTF8(s string, maxBytes int) string {
|
||||
if len(s) <= maxBytes {
|
||||
return s
|
||||
}
|
||||
cut := maxBytes
|
||||
for cut > 0 && !utf8.RuneStart(s[cut]) {
|
||||
cut--
|
||||
}
|
||||
return s[:cut]
|
||||
}
|
||||
|
||||
// RecordError records a poll failure for a source: it stamps last_run_at and
|
||||
// stores the (length-bounded) error message without advancing the marker.
|
||||
func (r *Repository) RecordError(ctx context.Context, sourceID, msg string) error {
|
||||
msg = truncateUTF8(msg, maxLastErrorLen)
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE autoscan_sources
|
||||
SET last_error = $2, last_run_at = now(), updated_at = now()
|
||||
WHERE id = $1`, sourceID, msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("record autoscan error: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: source %s", ErrNotFound, sourceID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const eventColumns = `id, source_id, installation_id, capability_id, started_at, completed_at,
|
||||
duration_ms, status, changes_returned, changes_resolved, targets_claimed, scans_created,
|
||||
scans_reused, scans_suppressed, error_message, marker_before, marker_after`
|
||||
|
||||
func scanEvent(row interface{ Scan(...any) error }) (Event, error) {
|
||||
var e Event
|
||||
var status string
|
||||
if err := row.Scan(
|
||||
&e.ID,
|
||||
&e.SourceID,
|
||||
&e.InstallationID,
|
||||
&e.CapabilityID,
|
||||
&e.StartedAt,
|
||||
&e.CompletedAt,
|
||||
&e.DurationMS,
|
||||
&status,
|
||||
&e.ChangesReturned,
|
||||
&e.ChangesResolved,
|
||||
&e.TargetsClaimed,
|
||||
&e.ScansCreated,
|
||||
&e.ScansReused,
|
||||
&e.ScansSuppressed,
|
||||
&e.ErrorMessage,
|
||||
&e.MarkerBefore,
|
||||
&e.MarkerAfter,
|
||||
); err != nil {
|
||||
return Event{}, err
|
||||
}
|
||||
e.Status = EventStatus(status)
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func (r *Repository) CreateEvent(ctx context.Context, in EventCreate) (int64, error) {
|
||||
started := in.StartedAt
|
||||
if started.IsZero() {
|
||||
started = time.Now()
|
||||
}
|
||||
row := r.pool.QueryRow(ctx, `
|
||||
INSERT INTO autoscan_events (
|
||||
source_id, installation_id, capability_id, started_at, completed_at,
|
||||
duration_ms, status, error_message, marker_before
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $4, 0, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
nullable(in.SourceID),
|
||||
in.InstallationID,
|
||||
in.CapabilityID,
|
||||
started,
|
||||
string(EventStatusError),
|
||||
"poll started but did not finish",
|
||||
nullable(in.MarkerBefore),
|
||||
)
|
||||
var id int64
|
||||
if err := row.Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("create autoscan event: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (r *Repository) FinishEvent(ctx context.Context, in EventFinish) error {
|
||||
if in.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
completed := in.CompletedAt
|
||||
if completed.IsZero() {
|
||||
completed = time.Now()
|
||||
}
|
||||
msg := truncateUTF8(in.ErrorMessage, maxLastErrorLen)
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE autoscan_events
|
||||
SET completed_at = $2,
|
||||
duration_ms = GREATEST(0, EXTRACT(EPOCH FROM ($2 - started_at)) * 1000)::bigint,
|
||||
status = $3,
|
||||
changes_returned = $4,
|
||||
changes_resolved = $5,
|
||||
targets_claimed = $6,
|
||||
scans_created = $7,
|
||||
scans_reused = $8,
|
||||
scans_suppressed = $9,
|
||||
error_message = $10,
|
||||
marker_after = $11
|
||||
WHERE id = $1`,
|
||||
in.ID,
|
||||
completed,
|
||||
string(in.Status),
|
||||
in.ChangesReturned,
|
||||
in.ChangesResolved,
|
||||
in.TargetsClaimed,
|
||||
in.ScansCreated,
|
||||
in.ScansReused,
|
||||
in.ScansSuppressed,
|
||||
msg,
|
||||
nullable(in.MarkerAfter),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finish autoscan event: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: autoscan event %d", ErrNotFound, in.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clampAutoscanLimit bounds a requested page size to a sane window: the
|
||||
// default page when unset, capped so a single query can never fan out.
|
||||
func clampAutoscanLimit(limit int) int {
|
||||
if limit <= 0 {
|
||||
return 50
|
||||
}
|
||||
if limit > 200 {
|
||||
return 200
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
// eventFilterClauses builds the shared WHERE clauses (and positional args) for
|
||||
// autoscan event queries, so listing and counting filter identically and the
|
||||
// search SQL lives in exactly one place.
|
||||
func eventFilterClauses(filter EventListFilter) ([]string, []any) {
|
||||
clauses := []string{"true"}
|
||||
args := []any{}
|
||||
if strings.TrimSpace(filter.SourceID) != "" {
|
||||
args = append(args, strings.TrimSpace(filter.SourceID))
|
||||
clauses = append(clauses, fmt.Sprintf("source_id = $%d", len(args)))
|
||||
}
|
||||
if filter.Status != "" {
|
||||
args = append(args, string(filter.Status))
|
||||
clauses = append(clauses, fmt.Sprintf("status = $%d", len(args)))
|
||||
}
|
||||
if search := strings.ToLower(strings.TrimSpace(filter.Search)); search != "" {
|
||||
args = append(args, "%"+search+"%")
|
||||
param := fmt.Sprintf("$%d", len(args))
|
||||
clauses = append(clauses, `(
|
||||
lower(capability_id) LIKE `+param+`
|
||||
OR lower(status) LIKE `+param+`
|
||||
OR lower(error_message) LIKE `+param+`
|
||||
OR lower(COALESCE(source_id::text, '')) LIKE `+param+`
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM scan_runs sr
|
||||
WHERE sr.autoscan_event_id = autoscan_events.id
|
||||
AND (
|
||||
lower(sr.id) LIKE `+param+`
|
||||
OR lower(sr.mode) LIKE `+param+`
|
||||
OR lower(sr.path) LIKE `+param+`
|
||||
OR lower(sr.status) LIKE `+param+`
|
||||
OR lower(COALESCE(sr.error_message, '')) LIKE `+param+`
|
||||
)
|
||||
)
|
||||
)`)
|
||||
}
|
||||
return clauses, args
|
||||
}
|
||||
|
||||
// scanFilterClauses builds the shared WHERE clauses for autoscan scan queries.
|
||||
// The clauses reference the `sr` (scan_runs) and `e` (autoscan_events) aliases,
|
||||
// so callers must select FROM scan_runs sr LEFT JOIN autoscan_events e.
|
||||
func scanFilterClauses(filter ScanListFilter) ([]string, []any) {
|
||||
clauses := []string{"sr.trigger = 'autoscan'"}
|
||||
args := []any{}
|
||||
if strings.TrimSpace(filter.Status) != "" {
|
||||
args = append(args, strings.TrimSpace(filter.Status))
|
||||
clauses = append(clauses, fmt.Sprintf("sr.status = $%d", len(args)))
|
||||
}
|
||||
if search := strings.ToLower(strings.TrimSpace(filter.Search)); search != "" {
|
||||
args = append(args, "%"+search+"%")
|
||||
param := fmt.Sprintf("$%d", len(args))
|
||||
clauses = append(clauses, `(
|
||||
lower(sr.id) LIKE `+param+`
|
||||
OR lower(sr.mode) LIKE `+param+`
|
||||
OR lower(sr.path) LIKE `+param+`
|
||||
OR lower(sr.status) LIKE `+param+`
|
||||
OR lower(COALESCE(sr.error_message, '')) LIKE `+param+`
|
||||
OR lower(COALESCE(e.capability_id, '')) LIKE `+param+`
|
||||
OR lower(COALESCE(e.status, '')) LIKE `+param+`
|
||||
OR lower(COALESCE(e.source_id::text, '')) LIKE `+param+`
|
||||
)`)
|
||||
}
|
||||
return clauses, args
|
||||
}
|
||||
|
||||
// CountEvents returns the total number of autoscan events matching filter,
|
||||
// ignoring limit/offset. It powers the "of N" total in paginated views.
|
||||
func (r *Repository) CountEvents(ctx context.Context, filter EventListFilter) (int, error) {
|
||||
clauses, args := eventFilterClauses(filter)
|
||||
var total int
|
||||
if err := r.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM autoscan_events
|
||||
WHERE `+strings.Join(clauses, " AND "),
|
||||
args...,
|
||||
).Scan(&total); err != nil {
|
||||
return 0, fmt.Errorf("count autoscan events: %w", err)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// CountAutoscanScans returns the total number of autoscan scan runs matching
|
||||
// filter, ignoring limit/offset. It powers the "of N" total in paginated views.
|
||||
func (r *Repository) CountAutoscanScans(ctx context.Context, filter ScanListFilter) (int, error) {
|
||||
clauses, args := scanFilterClauses(filter)
|
||||
var total int
|
||||
if err := r.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM scan_runs sr
|
||||
LEFT JOIN autoscan_events e ON e.id = sr.autoscan_event_id
|
||||
WHERE `+strings.Join(clauses, " AND "),
|
||||
args...,
|
||||
).Scan(&total); err != nil {
|
||||
return 0, fmt.Errorf("count autoscan scans: %w", err)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (r *Repository) ListEvents(ctx context.Context, filter EventListFilter) ([]EventWithRuns, error) {
|
||||
limit := clampAutoscanLimit(filter.Limit)
|
||||
offset := filter.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
clauses, args := eventFilterClauses(filter)
|
||||
args = append(args, limit)
|
||||
limitParam := len(args)
|
||||
args = append(args, offset)
|
||||
offsetParam := len(args)
|
||||
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT `+eventColumns+`
|
||||
FROM autoscan_events
|
||||
WHERE `+strings.Join(clauses, " AND ")+`
|
||||
ORDER BY completed_at DESC, id DESC
|
||||
LIMIT $`+fmt.Sprint(limitParam)+` OFFSET $`+fmt.Sprint(offsetParam),
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list autoscan events: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
events := make([]EventWithRuns, 0)
|
||||
ids := make([]int64, 0)
|
||||
indexByID := map[int64]int{}
|
||||
for rows.Next() {
|
||||
event, scanErr := scanEvent(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
indexByID[event.ID] = len(events)
|
||||
ids = append(ids, event.ID)
|
||||
events = append(events, EventWithRuns{Event: event, Runs: []ScanRunSummary{}})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return events, nil
|
||||
}
|
||||
|
||||
runRows, err := r.pool.Query(ctx, `
|
||||
SELECT autoscan_event_id, id, media_folder_id, mode, path, trigger, status,
|
||||
COALESCE(error_message, ''), requested_at, started_at, completed_at
|
||||
FROM scan_runs
|
||||
WHERE autoscan_event_id = ANY($1)
|
||||
ORDER BY requested_at ASC`,
|
||||
ids,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list autoscan event scan runs: %w", err)
|
||||
}
|
||||
defer runRows.Close()
|
||||
for runRows.Next() {
|
||||
var eventID int64
|
||||
var run ScanRunSummary
|
||||
if err := runRows.Scan(
|
||||
&eventID,
|
||||
&run.ID,
|
||||
&run.MediaFolderID,
|
||||
&run.Mode,
|
||||
&run.Path,
|
||||
&run.Trigger,
|
||||
&run.Status,
|
||||
&run.ErrorMessage,
|
||||
&run.RequestedAt,
|
||||
&run.StartedAt,
|
||||
&run.CompletedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if idx, ok := indexByID[eventID]; ok {
|
||||
events[idx].Runs = append(events[idx].Runs, run)
|
||||
}
|
||||
}
|
||||
return events, runRows.Err()
|
||||
}
|
||||
|
||||
func (r *Repository) ListAutoscanScans(ctx context.Context, filter ScanListFilter) ([]ScanWithEvent, error) {
|
||||
limit := clampAutoscanLimit(filter.Limit)
|
||||
offset := filter.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
clauses, args := scanFilterClauses(filter)
|
||||
args = append(args, limit)
|
||||
limitParam := len(args)
|
||||
args = append(args, offset)
|
||||
offsetParam := len(args)
|
||||
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT
|
||||
sr.id,
|
||||
sr.media_folder_id,
|
||||
sr.mode,
|
||||
sr.path,
|
||||
sr.trigger,
|
||||
sr.status,
|
||||
COALESCE(sr.error_message, ''),
|
||||
sr.requested_at,
|
||||
sr.started_at,
|
||||
sr.completed_at,
|
||||
sr.autoscan_event_id,
|
||||
e.source_id,
|
||||
e.installation_id,
|
||||
COALESCE(e.capability_id, ''),
|
||||
COALESCE(e.status, ''),
|
||||
e.completed_at
|
||||
FROM scan_runs sr
|
||||
LEFT JOIN autoscan_events e ON e.id = sr.autoscan_event_id
|
||||
WHERE `+strings.Join(clauses, " AND ")+`
|
||||
ORDER BY COALESCE(sr.completed_at, sr.started_at, sr.requested_at) DESC, sr.id DESC
|
||||
LIMIT $`+fmt.Sprint(limitParam)+` OFFSET $`+fmt.Sprint(offsetParam),
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list autoscan scans: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
scans := make([]ScanWithEvent, 0)
|
||||
for rows.Next() {
|
||||
var scan ScanWithEvent
|
||||
var eventStatus string
|
||||
if err := rows.Scan(
|
||||
&scan.ID,
|
||||
&scan.MediaFolderID,
|
||||
&scan.Mode,
|
||||
&scan.Path,
|
||||
&scan.Trigger,
|
||||
&scan.Status,
|
||||
&scan.ErrorMessage,
|
||||
&scan.RequestedAt,
|
||||
&scan.StartedAt,
|
||||
&scan.CompletedAt,
|
||||
&scan.AutoscanEventID,
|
||||
&scan.SourceID,
|
||||
&scan.InstallationID,
|
||||
&scan.CapabilityID,
|
||||
&eventStatus,
|
||||
&scan.EventCompletedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scan.EventStatus = EventStatus(eventStatus)
|
||||
scans = append(scans, scan)
|
||||
}
|
||||
return scans, rows.Err()
|
||||
}
|
||||
|
||||
func (r *Repository) GetQueueSummary(ctx context.Context) (QueueSummary, error) {
|
||||
var summary QueueSummary
|
||||
err := r.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE status = ANY($1))::int,
|
||||
COUNT(*) FILTER (WHERE status = $2)::int,
|
||||
COUNT(*) FILTER (WHERE status = $3)::int
|
||||
FROM scan_runs
|
||||
WHERE trigger = $4`,
|
||||
[]string{"accepted", "running"},
|
||||
"accepted",
|
||||
"running",
|
||||
"autoscan",
|
||||
).Scan(&summary.Active, &summary.Accepted, &summary.Running)
|
||||
if err != nil {
|
||||
return QueueSummary{}, fmt.Errorf("get autoscan queue summary: %w", err)
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (r *Repository) LatestEventAt(ctx context.Context) (*time.Time, error) {
|
||||
var latest *time.Time
|
||||
err := r.pool.QueryRow(ctx, `SELECT max(completed_at) FROM autoscan_events`).Scan(&latest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get latest autoscan event time: %w", err)
|
||||
}
|
||||
return latest, nil
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package autoscan
|
||||
|
||||
import "strings"
|
||||
|
||||
// normalizeSeparators converts Windows backslash separators to forward slashes
|
||||
// so paths from a Windows-hosted arr resolve on the Linux host. (filepath.ToSlash
|
||||
// is a no-op on Linux, so the replacement is explicit.)
|
||||
func normalizeSeparators(path string) string {
|
||||
return strings.ReplaceAll(path, "\\", "/")
|
||||
}
|
||||
|
||||
// applyRewrites returns path with the MOST-SPECIFIC matching prefix rewrite
|
||||
// applied, or path unchanged when none match.
|
||||
//
|
||||
// "Most-specific" means the longest matching From wins, not the first one in the
|
||||
// slice. A first-match strategy lets a broad rewrite (From="/data") shadow a
|
||||
// nested, more-specific one (From="/data/media") when the broad rule happens to
|
||||
// be listed first; the arr plugin review flagged exactly this. Selecting the
|
||||
// longest matching prefix makes the result independent of rule ordering.
|
||||
func applyRewrites(path string, rewrites []PathRewrite) string {
|
||||
bestIdx := -1
|
||||
bestLen := -1
|
||||
var bestTrimmed string
|
||||
for i, rw := range rewrites {
|
||||
// Normalize the stored From the SAME way coveredBy/normalizePath does
|
||||
// (backslash→slash, collapse '//', strip trailing '/') so a Windows-style
|
||||
// or dup-slash rewrite that suggest-time reports as "covered" actually
|
||||
// matches at poll time. Without this, a From like `D:\data\tv` would be
|
||||
// "covered" in the suggester yet never match the normalized incoming path.
|
||||
trimmed := normalizePath(rw.From)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
// Match only at a path-segment boundary: exact, or prefix followed by '/'.
|
||||
// This prevents From="/data/media" from rewriting "/data/media2/x".
|
||||
if path == trimmed || strings.HasPrefix(path, trimmed+"/") {
|
||||
if len(trimmed) > bestLen {
|
||||
bestLen = len(trimmed)
|
||||
bestIdx = i
|
||||
bestTrimmed = trimmed
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestIdx < 0 {
|
||||
return path
|
||||
}
|
||||
return strings.TrimSpace(rewrites[bestIdx].To) + strings.TrimPrefix(path, bestTrimmed)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package autoscan
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestApplyRewrites(t *testing.T) {
|
||||
rw := []PathRewrite{{From: "/data/media", To: "/mnt/media"}}
|
||||
cases := []struct{ in, want string }{
|
||||
{"/data/media/Movies/Dune/Dune.mkv", "/mnt/media/Movies/Dune/Dune.mkv"},
|
||||
{"/other/path/file.mkv", "/other/path/file.mkv"}, // no-match pass-through
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := applyRewrites(tc.in, rw); got != tc.want {
|
||||
t.Fatalf("applyRewrites(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Exact match (no trailing segment) rewrites the whole path.
|
||||
if got := applyRewrites("/data/media", rw); got != "/mnt/media" {
|
||||
t.Fatalf("exact match: got %q want %q", got, "/mnt/media")
|
||||
}
|
||||
|
||||
// nil/empty rewrites pass through unchanged.
|
||||
if got := applyRewrites("/data/media/x", nil); got != "/data/media/x" {
|
||||
t.Fatalf("nil rewrites: got %q", got)
|
||||
}
|
||||
|
||||
// Segment-boundary matching: a sibling dir sharing the prefix must NOT match.
|
||||
boundary := []PathRewrite{{From: "/data/media", To: "/mnt"}}
|
||||
if got := applyRewrites("/data/media2/x", boundary); got != "/data/media2/x" {
|
||||
t.Fatalf("boundary: /data/media2/x must not rewrite, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyRewritesNormalizesStoredFrom verifies that a Windows-style / dup-slash
|
||||
// stored From is normalized the same way coveredBy/normalizePath does, so a
|
||||
// rewrite the suggester reports as "covered" actually matches at poll time. The
|
||||
// incoming path is already separator-normalized by PollOnce before applyRewrites.
|
||||
func TestApplyRewritesNormalizesStoredFrom(t *testing.T) {
|
||||
// Backslash From: a Windows-hosted arr root stored verbatim.
|
||||
winFrom := []PathRewrite{{From: `D:\data\tv`, To: "/mnt/media/tv"}}
|
||||
if got := applyRewrites("D:/data/tv/Show/S01/E01.mkv", winFrom); got != "/mnt/media/tv/Show/S01/E01.mkv" {
|
||||
t.Fatalf("backslash From should match normalized path, got %q", got)
|
||||
}
|
||||
// coveredBy must agree with applyRewrites: the normalized root is covered.
|
||||
if !coveredBy("D:/data/tv", winFrom) {
|
||||
t.Fatalf("coveredBy should report the backslash From as covering the root")
|
||||
}
|
||||
|
||||
// Dup-slash From collapses too.
|
||||
dupFrom := []PathRewrite{{From: "/data//tv/", To: "/mnt/media/tv"}}
|
||||
if got := applyRewrites("/data/tv/Show/E.mkv", dupFrom); got != "/mnt/media/tv/Show/E.mkv" {
|
||||
t.Fatalf("dup-slash From should match collapsed path, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyRewritesMostSpecificWins verifies the longest matching From wins
|
||||
// regardless of slice ordering: a broad rule must not shadow a nested one.
|
||||
func TestApplyRewritesMostSpecificWins(t *testing.T) {
|
||||
// Broad rule listed FIRST: a first-match strategy would (wrongly) pick /data
|
||||
// and yield "/A/media/x". Most-specific must pick /data/media -> "/B/x".
|
||||
broadFirst := []PathRewrite{
|
||||
{From: "/data", To: "/A"},
|
||||
{From: "/data/media", To: "/B"},
|
||||
}
|
||||
if got := applyRewrites("/data/media/x", broadFirst); got != "/B/x" {
|
||||
t.Fatalf("broad-first: got %q want %q", got, "/B/x")
|
||||
}
|
||||
|
||||
// Same rules, specific listed first: result must be identical (order-independent).
|
||||
specificFirst := []PathRewrite{
|
||||
{From: "/data/media", To: "/B"},
|
||||
{From: "/data", To: "/A"},
|
||||
}
|
||||
if got := applyRewrites("/data/media/x", specificFirst); got != "/B/x" {
|
||||
t.Fatalf("specific-first: got %q want %q", got, "/B/x")
|
||||
}
|
||||
|
||||
// A path under the broad rule but NOT the specific one still uses the broad rule.
|
||||
if got := applyRewrites("/data/other/x", broadFirst); got != "/A/other/x" {
|
||||
t.Fatalf("broad fallthrough: got %q want %q", got, "/A/other/x")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSeparators(t *testing.T) {
|
||||
if got := normalizeSeparators(`C:\Media\Movies\Dune\Dune.mkv`); got != "C:/Media/Movies/Dune/Dune.mkv" {
|
||||
t.Fatalf("normalizeSeparators(windows) = %q", got)
|
||||
}
|
||||
// POSIX paths are unchanged.
|
||||
if got := normalizeSeparators("/mnt/media/x.mkv"); got != "/mnt/media/x.mkv" {
|
||||
t.Fatalf("normalizeSeparators(posix) = %q", got)
|
||||
}
|
||||
// A normalized Windows path then rewrites on the Linux host.
|
||||
rw := []PathRewrite{{From: "C:/Media", To: "/mnt/media"}}
|
||||
if got := applyRewrites(normalizeSeparators(`C:\Media\ShowA\S01\E01.mkv`), rw); got != "/mnt/media/ShowA/S01/E01.mkv" {
|
||||
t.Fatalf("windows rewrite = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,464 @@
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/scantrigger"
|
||||
)
|
||||
|
||||
const scanTrigger = "autoscan"
|
||||
|
||||
// Store is the persistence surface the engine needs. The repository implements
|
||||
// the full CRUD surface; this interface is the read/bookkeeping subset the poll
|
||||
// loop touches.
|
||||
type Store interface {
|
||||
GetSettings(ctx context.Context) (Settings, error)
|
||||
ListEnabledSources(ctx context.Context) ([]Source, error)
|
||||
GetSource(ctx context.Context, id string) (Source, error)
|
||||
GetConnection(ctx context.Context, id string) (Connection, error)
|
||||
AdvanceMarker(ctx context.Context, sourceID, marker string) error
|
||||
RecordError(ctx context.Context, sourceID, msg string) error
|
||||
CreateEvent(ctx context.Context, event EventCreate) (int64, error)
|
||||
FinishEvent(ctx context.Context, event EventFinish) error
|
||||
}
|
||||
|
||||
// Resolver maps a Silo-native path to a concrete scan target (library folder).
|
||||
type Resolver interface {
|
||||
Resolve(ctx context.Context, req scantrigger.Request) (*scantrigger.Target, error)
|
||||
ResolveMissingSubtree(ctx context.Context, subtreePath, trigger string) (*scantrigger.Target, error)
|
||||
}
|
||||
|
||||
// Queuer enqueues resolved scan targets.
|
||||
type Queuer interface {
|
||||
EnqueueScans(ctx context.Context, targets []scantrigger.Target) error
|
||||
EnqueueAutoscanScans(ctx context.Context, targets []scantrigger.Target, eventID int64) (created, reused int, err error)
|
||||
}
|
||||
|
||||
type resolveStats struct {
|
||||
ChangesResolved int
|
||||
TargetsClaimed int
|
||||
Suppressed int
|
||||
}
|
||||
|
||||
// connectionResolver resolves a stored connection to concrete credentials.
|
||||
type connectionResolver interface {
|
||||
Resolve(ctx context.Context, c Connection) (ResolvedConnection, error)
|
||||
}
|
||||
|
||||
// RootFolderClient lists a Radarr/Sonarr instance's configured root folder
|
||||
// paths (used by the rewrite suggester). The concrete impl additionally
|
||||
// satisfies ArrStatusProbe for the connection-test endpoint.
|
||||
type RootFolderClient interface {
|
||||
RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error)
|
||||
}
|
||||
|
||||
// FolderLister lists every Silo media-folder path (used by the rewrite
|
||||
// suggester to match arr roots against Silo folders).
|
||||
type FolderLister interface {
|
||||
ListFolderPaths(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
store Store
|
||||
provider ScanSourceProvider
|
||||
connres connectionResolver
|
||||
resolver Resolver
|
||||
queue Queuer
|
||||
suppress Suppressor
|
||||
lister ScanSourceLister
|
||||
|
||||
// Optional deps for the connection-test and rewrite-suggester endpoints.
|
||||
// Wired via setters so the poll-loop constructor stays unchanged and tests
|
||||
// that only exercise PollOnce need not supply them.
|
||||
rootFolders RootFolderClient
|
||||
folders FolderLister
|
||||
}
|
||||
|
||||
// SetSuggesterDeps wires the dependencies the rewrite-suggester and
|
||||
// connection-test endpoints need: an arr root-folder/status client and a Silo
|
||||
// media-folder lister. Optional — when unset, SuggestRewrites/TestConnection
|
||||
// return an error.
|
||||
func (s *Service) SetSuggesterDeps(rootFolders RootFolderClient, folders FolderLister) {
|
||||
s.rootFolders = rootFolders
|
||||
s.folders = folders
|
||||
}
|
||||
|
||||
// NewService builds the autoscan engine. The provider supplies changed paths
|
||||
// from scan_source plugins; connres resolves a source's connection to concrete
|
||||
// credentials; resolver/queue/suppress drive the resolve→suppress→enqueue loop.
|
||||
// lister enumerates installed scan_source capabilities so the Add-source picker
|
||||
// can offer them and PollOnce can skip orphaned source rows; it may be nil
|
||||
// (the picker then returns an empty list and orphan-pruning is disabled).
|
||||
func NewService(
|
||||
store Store,
|
||||
provider ScanSourceProvider,
|
||||
connres connectionResolver,
|
||||
resolver Resolver,
|
||||
queue Queuer,
|
||||
suppress Suppressor,
|
||||
lister ScanSourceLister,
|
||||
) *Service {
|
||||
return &Service{
|
||||
store: store,
|
||||
provider: provider,
|
||||
connres: connres,
|
||||
resolver: resolver,
|
||||
queue: queue,
|
||||
suppress: suppress,
|
||||
lister: lister,
|
||||
}
|
||||
}
|
||||
|
||||
// PollOnce runs one autoscan cycle. Per-source failures are logged and skipped;
|
||||
// only settings/listing errors propagate. The opaque next marker returned by the
|
||||
// provider is stored verbatim, but only when the cycle's work is genuinely
|
||||
// consumed: the provider returned no paths, or it returned paths and at least one
|
||||
// resolved+enqueued. When paths come back but NONE resolve to a library folder
|
||||
// (e.g. a freshly-enabled source with unconfigured rewrites) the marker is held
|
||||
// and an error recorded, so those imports aren't skipped forever.
|
||||
func (s *Service) PollOnce(ctx context.Context) error {
|
||||
// Fetch the set of currently-installed scan_source capabilities so we can skip
|
||||
// orphaned source rows (their plugin was uninstalled). Listing failures are
|
||||
// non-fatal: a nil set means the installed set is unavailable, in which case
|
||||
// we do NOT prune orphans — every enabled source is assumed present.
|
||||
present, derr := s.installedScanSources(ctx)
|
||||
if derr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: list installed scan sources failed", "err", derr)
|
||||
present = nil
|
||||
}
|
||||
|
||||
settings, err := s.store.GetSettings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !settings.Enabled {
|
||||
return nil
|
||||
}
|
||||
sources, err := s.store.ListEnabledSources(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ttl := time.Duration(settings.DebounceSeconds) * time.Second
|
||||
now := time.Now()
|
||||
|
||||
for _, src := range sources {
|
||||
// Skip orphaned sources: an enabled source whose scan_source plugin has
|
||||
// been uninstalled/disabled is no longer in the discovered set, so polling
|
||||
// it would error every cycle. Skip it quietly (no RecordError) so the spam
|
||||
// stops; the operator can delete it via the source delete endpoint. A nil
|
||||
// `present` set means discovery is unavailable — don't prune in that case.
|
||||
if present != nil {
|
||||
if _, ok := present[installedKey{InstallationID: src.InstallationID, CapabilityID: src.CapabilityID}]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Honor the per-source poll interval as a "poll at most every N seconds"
|
||||
// floor: the global task fires at the default cadence, so a source with a
|
||||
// longer interval is skipped until enough time has elapsed.
|
||||
interval := time.Duration(settings.DefaultPollIntervalSeconds) * time.Second
|
||||
if src.PollIntervalSeconds != nil {
|
||||
interval = time.Duration(*src.PollIntervalSeconds) * time.Second
|
||||
}
|
||||
if src.LastRunAt != nil && now.Sub(*src.LastRunAt) < interval {
|
||||
continue
|
||||
}
|
||||
marker := ""
|
||||
if src.Marker != nil {
|
||||
marker = *src.Marker
|
||||
}
|
||||
eventID := s.createEvent(ctx, src, marker, time.Now())
|
||||
// A connection is OPTIONAL. Server-based providers (Sonarr/Radarr) bind a
|
||||
// connection and the resolved {base_url, api_key} is handed to the plugin.
|
||||
// Other providers (e.g. a filesystem/CephFS watcher) need none and get an
|
||||
// empty connection they ignore. If a plugin requires a connection it didn't
|
||||
// get, it returns an error that is RecordError'd below — so the operator
|
||||
// still sees "needs attention" without the host assuming every source is
|
||||
// credential-based.
|
||||
var conn ResolvedConnection
|
||||
if src.ConnectionID != nil {
|
||||
resolved, cerr := s.resolveConnection(ctx, *src.ConnectionID)
|
||||
if cerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: resolve connection failed", "source_id", src.ID, "err", cerr)
|
||||
if rerr := s.store.RecordError(ctx, src.ID, cerr.Error()); rerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: record error failed", "source_id", src.ID, "err", rerr)
|
||||
}
|
||||
s.finishEvent(ctx, eventID, EventFinish{
|
||||
Status: EventStatusError,
|
||||
ErrorMessage: cerr.Error(),
|
||||
MarkerAfter: marker,
|
||||
})
|
||||
continue
|
||||
}
|
||||
conn = resolved
|
||||
}
|
||||
changes, next, perr := s.provider.PollChanges(ctx, src.InstallationID, src.CapabilityID, marker, conn, src.SourceConfig)
|
||||
if perr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: poll changes failed", "source_id", src.ID, "err", perr)
|
||||
if rerr := s.store.RecordError(ctx, src.ID, perr.Error()); rerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: record error failed", "source_id", src.ID, "err", rerr)
|
||||
}
|
||||
s.finishEvent(ctx, eventID, EventFinish{
|
||||
Status: EventStatusError,
|
||||
ErrorMessage: perr.Error(),
|
||||
MarkerAfter: marker,
|
||||
})
|
||||
continue // do NOT advance marker
|
||||
}
|
||||
|
||||
rewritten := rewriteChanges(changes, src.PathRewrites)
|
||||
targets, claimed, resolvedAny, stats := s.resolveAndClaim(ctx, rewritten, ttl)
|
||||
var enqueue EnqueueResult
|
||||
if len(targets) > 0 {
|
||||
var eerr error
|
||||
if eventID != 0 {
|
||||
enqueue.Created, enqueue.Reused, eerr = s.queue.EnqueueAutoscanScans(ctx, targets, eventID)
|
||||
} else {
|
||||
eerr = s.queue.EnqueueScans(ctx, targets)
|
||||
enqueue.Created = len(targets)
|
||||
}
|
||||
if eerr != nil {
|
||||
s.releaseClaims(ctx, claimed)
|
||||
slog.WarnContext(ctx, "autoscan: enqueue failed", "source_id", src.ID, "err", eerr)
|
||||
s.finishEvent(ctx, eventID, EventFinish{
|
||||
Status: EventStatusError,
|
||||
ChangesReturned: len(changes),
|
||||
ChangesResolved: stats.ChangesResolved,
|
||||
TargetsClaimed: stats.TargetsClaimed,
|
||||
ScansSuppressed: stats.Suppressed,
|
||||
ErrorMessage: eerr.Error(),
|
||||
MarkerAfter: marker,
|
||||
})
|
||||
continue // do NOT advance marker
|
||||
}
|
||||
}
|
||||
|
||||
// Advancing the marker is what tells the provider "I've consumed up to
|
||||
// here". Advance it ONLY when the work it represents is genuinely done:
|
||||
// - provider returned ZERO paths → nothing to do, advance normally.
|
||||
// - returned paths AND ≥1 resolved → enqueued (above), advance.
|
||||
// - returned paths, resolved but all
|
||||
// suppressed (recently scanned /
|
||||
// debounced) → work is effectively done; advance.
|
||||
// - returned paths AND NOTHING resolved → do NOT advance. This is the
|
||||
// freshly-enabled-source state (path_rewrites not configured yet); the
|
||||
// incoming paths don't map to any Silo library folder. Advancing here
|
||||
// would skip those imports forever. Surface why so the operator can fix
|
||||
// the rewrites, then a later poll re-reads the same window and resolves.
|
||||
// (Some-but-not-all resolving counts as resolved — the unresolved paths are
|
||||
// legitimately outside Silo's libraries, so advancing is correct.)
|
||||
//
|
||||
// NOTE: len(targets)==0 alone is NOT misconfiguration — paths can resolve
|
||||
// yet be fully suppressed. Gate the error on resolvedAny, not targets.
|
||||
if len(changes) > 0 && !resolvedAny {
|
||||
msg := fmt.Sprintf("returned %d path(s) but none matched a Silo library folder — check this source's path rewrites", len(changes))
|
||||
if rerr := s.store.RecordError(ctx, src.ID, msg); rerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: record error failed", "source_id", src.ID, "err", rerr)
|
||||
}
|
||||
s.finishEvent(ctx, eventID, EventFinish{
|
||||
Status: EventStatusUnresolved,
|
||||
ChangesReturned: len(changes),
|
||||
ErrorMessage: msg,
|
||||
MarkerAfter: marker,
|
||||
})
|
||||
continue // do NOT advance marker
|
||||
}
|
||||
if aerr := s.store.AdvanceMarker(ctx, src.ID, next); aerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: advance marker failed", "source_id", src.ID, "err", aerr)
|
||||
s.finishEvent(ctx, eventID, EventFinish{
|
||||
Status: EventStatusError,
|
||||
ChangesReturned: len(changes),
|
||||
ChangesResolved: stats.ChangesResolved,
|
||||
TargetsClaimed: stats.TargetsClaimed,
|
||||
ScansCreated: enqueue.Created,
|
||||
ScansReused: enqueue.Reused,
|
||||
ScansSuppressed: stats.Suppressed,
|
||||
ErrorMessage: aerr.Error(),
|
||||
MarkerAfter: marker,
|
||||
})
|
||||
continue
|
||||
}
|
||||
s.finishEvent(ctx, eventID, EventFinish{
|
||||
Status: EventStatusSuccess,
|
||||
ChangesReturned: len(changes),
|
||||
ChangesResolved: stats.ChangesResolved,
|
||||
TargetsClaimed: stats.TargetsClaimed,
|
||||
ScansCreated: enqueue.Created,
|
||||
ScansReused: enqueue.Reused,
|
||||
ScansSuppressed: stats.Suppressed,
|
||||
MarkerAfter: next,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) createEvent(ctx context.Context, src Source, marker string, startedAt time.Time) int64 {
|
||||
if s == nil || s.store == nil {
|
||||
return 0
|
||||
}
|
||||
id, err := s.store.CreateEvent(ctx, EventCreate{
|
||||
SourceID: src.ID,
|
||||
InstallationID: src.InstallationID,
|
||||
CapabilityID: src.CapabilityID,
|
||||
StartedAt: startedAt,
|
||||
MarkerBefore: marker,
|
||||
})
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "autoscan: create event failed", "source_id", src.ID, "err", err)
|
||||
return 0
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (s *Service) finishEvent(ctx context.Context, eventID int64, finish EventFinish) {
|
||||
if eventID == 0 || s == nil || s.store == nil {
|
||||
return
|
||||
}
|
||||
finish.ID = eventID
|
||||
finish.CompletedAt = time.Now()
|
||||
if finish.Status == "" {
|
||||
finish.Status = EventStatusSuccess
|
||||
}
|
||||
if err := s.store.FinishEvent(ctx, finish); err != nil {
|
||||
slog.WarnContext(ctx, "autoscan: finish event failed", "event_id", eventID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveConnection loads and resolves a source's connection to credentials.
|
||||
func (s *Service) resolveConnection(ctx context.Context, connectionID string) (ResolvedConnection, error) {
|
||||
conn, err := s.store.GetConnection(ctx, connectionID)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, err
|
||||
}
|
||||
return s.connres.Resolve(ctx, conn)
|
||||
}
|
||||
|
||||
func rewriteChanges(changes []Change, rewrites []PathRewrite) []Change {
|
||||
rewritten := make([]Change, 0, len(changes))
|
||||
for _, change := range changes {
|
||||
path := applyRewrites(normalizeSeparators(change.SourcePath), rewrites)
|
||||
rewritten = append(rewritten, Change{SourcePath: path, Scope: change.Scope})
|
||||
}
|
||||
return rewritten
|
||||
}
|
||||
|
||||
// resolveAndClaim resolves changes to scan targets and atomically claims them
|
||||
// via the suppressor. Legacy/auto changes retain the historical parent-dir
|
||||
// collapse. Structured file changes resolve exact files, and subtree changes
|
||||
// resolve exact subtree paths even when the path no longer exists.
|
||||
func (s *Service) resolveAndClaim(ctx context.Context, changes []Change, ttl time.Duration) (targets []scantrigger.Target, claimed []string, resolvedAny bool, stats resolveStats) {
|
||||
seenTargets := make(map[string]struct{})
|
||||
|
||||
var legacyPaths []string
|
||||
for _, change := range changes {
|
||||
switch change.Scope {
|
||||
case ChangeScopeFile, ChangeScopeSubtree:
|
||||
target, ok := s.resolveChange(ctx, change)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
resolvedAny = true
|
||||
stats.ChangesResolved++
|
||||
if !s.claimTarget(ctx, *target, ttl, seenTargets, &targets, &claimed) {
|
||||
stats.Suppressed++
|
||||
}
|
||||
default:
|
||||
legacyPaths = append(legacyPaths, change.SourcePath)
|
||||
}
|
||||
}
|
||||
|
||||
for _, dir := range uniqueParentDirs(legacyPaths) {
|
||||
target, rerr := s.resolver.Resolve(ctx, scantrigger.Request{Path: dir, Trigger: scanTrigger})
|
||||
if rerr != nil {
|
||||
var reqErr *scantrigger.RequestError
|
||||
if errors.As(rerr, &reqErr) {
|
||||
// Path outside Silo's media folders (or otherwise unresolvable)
|
||||
// — an expected skip, not an error worth logging every cycle.
|
||||
continue
|
||||
}
|
||||
slog.WarnContext(ctx, "autoscan: resolve failed", "path", dir, "err", rerr)
|
||||
continue
|
||||
}
|
||||
if target == nil || target.Folder == nil {
|
||||
continue
|
||||
}
|
||||
resolvedAny = true
|
||||
stats.ChangesResolved++
|
||||
if !s.claimTarget(ctx, *target, ttl, seenTargets, &targets, &claimed) {
|
||||
stats.Suppressed++
|
||||
}
|
||||
}
|
||||
stats.TargetsClaimed = len(targets)
|
||||
return targets, claimed, resolvedAny, stats
|
||||
}
|
||||
|
||||
func (s *Service) resolveChange(ctx context.Context, change Change) (*scantrigger.Target, bool) {
|
||||
if change.SourcePath == "" {
|
||||
return nil, false
|
||||
}
|
||||
var (
|
||||
target *scantrigger.Target
|
||||
err error
|
||||
)
|
||||
switch change.Scope {
|
||||
case ChangeScopeSubtree:
|
||||
target, err = s.resolver.ResolveMissingSubtree(ctx, change.SourcePath, scanTrigger)
|
||||
case ChangeScopeFile:
|
||||
target, err = s.resolver.Resolve(ctx, scantrigger.Request{Path: change.SourcePath, Trigger: scanTrigger})
|
||||
if err == nil && target != nil && target.Mode != scantrigger.ModeFile {
|
||||
return nil, false
|
||||
}
|
||||
default:
|
||||
target, err = s.resolver.Resolve(ctx, scantrigger.Request{Path: change.SourcePath, Trigger: scanTrigger})
|
||||
}
|
||||
if err != nil {
|
||||
var reqErr *scantrigger.RequestError
|
||||
if errors.As(err, &reqErr) {
|
||||
return nil, false
|
||||
}
|
||||
slog.WarnContext(ctx, "autoscan: resolve failed", "path", change.SourcePath, "scope", change.Scope, "err", err)
|
||||
return nil, false
|
||||
}
|
||||
if target == nil || target.Folder == nil {
|
||||
return nil, false
|
||||
}
|
||||
return target, true
|
||||
}
|
||||
|
||||
func (s *Service) claimTarget(
|
||||
ctx context.Context,
|
||||
target scantrigger.Target,
|
||||
ttl time.Duration,
|
||||
seenTargets map[string]struct{},
|
||||
targets *[]scantrigger.Target,
|
||||
claimed *[]string,
|
||||
) bool {
|
||||
targetKey := fmt.Sprintf("%d|%s|%s", target.Folder.ID, target.Mode, target.Path)
|
||||
if _, seen := seenTargets[targetKey]; seen {
|
||||
return false
|
||||
}
|
||||
seenTargets[targetKey] = struct{}{}
|
||||
|
||||
key := fmt.Sprintf("%d|%s", target.Folder.ID, target.Path)
|
||||
ok, serr := s.suppress.ShouldScan(ctx, key, ttl)
|
||||
if serr != nil || !ok {
|
||||
return false
|
||||
}
|
||||
target.Trigger = scanTrigger
|
||||
*targets = append(*targets, target)
|
||||
*claimed = append(*claimed, key)
|
||||
return true
|
||||
}
|
||||
|
||||
// releaseClaims drops suppression claims (used when the scan enqueue fails so a
|
||||
// later cycle can retry the same targets).
|
||||
func (s *Service) releaseClaims(ctx context.Context, claimed []string) {
|
||||
for _, k := range claimed {
|
||||
if rerr := s.suppress.Release(ctx, k); rerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: release claim failed", "key", k, "err", rerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user