Compare commits
453
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
67209e38e0 | ||
|
|
35b8b9e44e | ||
|
|
be704c3a06 | ||
|
|
dcce156d4a | ||
|
|
dcb8ce8afe | ||
|
|
93eb23d2ec | ||
|
|
20b75732ac | ||
|
|
c47c3dc621 | ||
|
|
84704a073c | ||
|
|
e2865c070b | ||
|
|
fd6797d907 | ||
|
|
fede2a986f | ||
|
|
c2a33c7e35 | ||
|
|
0506336625 | ||
|
|
19a0e62f0d | ||
|
|
af575d7470 | ||
|
|
3b3e9ed7b9 | ||
|
|
d9ddfdb879 | ||
|
|
3c0fd5f897 | ||
|
|
d2c8be9162 | ||
|
|
9a1b4ca3d7 | ||
|
|
4eb8c06023 | ||
|
|
dd54e90894 | ||
|
|
abb080c6ed | ||
|
|
f8b7cb0605 | ||
|
|
ed1ab977b3 | ||
|
|
2fe7c8683c | ||
|
|
7d0154852d | ||
|
|
e70bccceb7 | ||
|
|
615d35c131 | ||
|
|
bbc865ce53 | ||
|
|
7649b30f77 | ||
|
|
088250f6ea | ||
|
|
ea049c3d1f | ||
|
|
0b8b67f6c4 | ||
|
|
bb620a16ab | ||
|
|
f9c5c13a1a | ||
|
|
06d11fa062 | ||
|
|
addeb47807 | ||
|
|
e7d7fd87d6 | ||
|
|
84f027aa07 | ||
|
|
afa6025a2a | ||
|
|
6a5f61dd39 | ||
|
|
0b96771964 | ||
|
|
497fd6ee71 | ||
|
|
bde9d563ab | ||
|
|
f25cbb27aa | ||
|
|
bf9746ce32 | ||
|
|
1fa00b784f | ||
|
|
4d3f366dd2 | ||
|
|
d5791ed541 | ||
|
|
0a52b47e4e | ||
|
|
418b7347ca | ||
|
|
75b4e1c72f | ||
|
|
6e817706f1 | ||
|
|
404f782ee2 | ||
|
|
ab7c081ecc | ||
|
|
01f828a11a | ||
|
|
a79eb85ca0 | ||
|
|
5711771c9e | ||
|
|
cdbfcbe9af | ||
|
|
ea92eb6c69 | ||
|
|
f7866a78d5 | ||
|
|
59cfb01e71 | ||
|
|
88f0228968 | ||
|
|
4be3f498cf | ||
|
|
ac4a44c62c | ||
|
|
a724ac9c55 | ||
|
|
abca92eec5 | ||
|
|
30f59accd7 | ||
|
|
c24df8b3bd | ||
|
|
1c823aa507 | ||
|
|
c7079000a8 | ||
|
|
9ce999a61d | ||
|
|
a638494540 | ||
|
|
b5b3cccc18 | ||
|
|
2e9435d69e | ||
|
|
f4fe1f7d94 | ||
|
|
1094ecb523 | ||
|
|
4f80010de5 | ||
|
|
229379e0a6 | ||
|
|
954d26360f | ||
|
|
09f202f9f7 | ||
|
|
833ce25178 | ||
|
|
a047519230 | ||
|
|
7c65c7b4dc | ||
|
|
98ed20444a | ||
|
|
e59343968d | ||
|
|
0add71fbd8 | ||
|
|
8d5e0fec85 | ||
|
|
423e2c02ae | ||
|
|
58fbc8f23c | ||
|
|
45995f659c | ||
|
|
1756f54e21 | ||
|
|
5150fec3e3 | ||
|
|
bf880b292b | ||
|
|
db22615874 | ||
|
|
cfbd95e31c | ||
|
|
70e8190348 | ||
|
|
9626fd8f13 | ||
|
|
0eb87fa0f5 | ||
|
|
fadd8ff456 | ||
|
|
a17529fd6f | ||
|
|
6f716d3404 | ||
|
|
7958f0bbf0 | ||
|
|
52dd5dff7f | ||
|
|
e64b130bd3 | ||
|
|
ed4cebf3ba | ||
|
|
39ba284c9d | ||
|
|
c7f65bf351 | ||
|
|
54b20f57d7 | ||
|
|
35a0db7d29 | ||
|
|
ef78a1c5da | ||
|
|
8b70357703 | ||
|
|
0bd4f8cb3b | ||
|
|
c5f21cb10d | ||
|
|
1a1282db0c | ||
|
|
79c749b9a7 | ||
|
|
46540dfec3 | ||
|
|
162e0cc449 | ||
|
|
7f8dc95eca | ||
|
|
2933e0581f | ||
|
|
074d106402 | ||
|
|
b246f271cb | ||
|
|
0d0d7bab1e | ||
|
|
ce9830cd02 | ||
|
|
75b476d124 | ||
|
|
2fe2c918f4 | ||
|
|
adbdd86ac5 | ||
|
|
b9a9613fd0 | ||
|
|
cb6272ef25 | ||
|
|
89f005dfa8 | ||
|
|
2c714e4ef2 | ||
|
|
66a4146fc2 | ||
|
|
b7439884ec | ||
|
|
7c64c06e3d | ||
|
|
2fa30379ad | ||
|
|
5855cebe75 | ||
|
|
cf4e080bf4 | ||
|
|
339ed20074 | ||
|
|
9a240edb17 | ||
|
|
9e29e7b330 | ||
|
|
d3626fca00 | ||
|
|
ed8f63c886 | ||
|
|
65ea9b651c | ||
|
|
3704b262d6 | ||
|
|
f5b3cba3cb | ||
|
|
96bae2f346 | ||
|
|
978e1b4954 | ||
|
|
c4e67effc3 | ||
|
|
6b1554a86c | ||
|
|
480ca44306 | ||
|
|
ecd85f9ca1 | ||
|
|
f9bb94a299 | ||
|
|
eb6024573e | ||
|
|
c469550fa0 | ||
|
|
3353456d46 | ||
|
|
826f0d0b9a | ||
|
|
0b3723c7f2 | ||
|
|
09204687fd | ||
|
|
1fb09efc9b | ||
|
|
064963f45f | ||
|
|
214171ac90 | ||
|
|
31b2716089 | ||
|
|
6564ee235a | ||
|
|
c16d876007 | ||
|
|
266b4453ca | ||
|
|
3ba4373e72 | ||
|
|
0163df3683 | ||
|
|
356c014f41 | ||
|
|
17c4f19db0 | ||
|
|
807384f4af | ||
|
|
c5daad0dcb | ||
|
|
1490b6318d | ||
|
|
1e36d42bcc | ||
|
|
6e5d096fdb | ||
|
|
9f73ac6f1a | ||
|
|
177cfdc485 | ||
|
|
efd377fbdc | ||
|
|
de89f0136c | ||
|
|
08a0704a11 | ||
|
|
cd946b30a3 | ||
|
|
dc0392dee0 | ||
|
|
681c846da8 | ||
|
|
54a04958b7 | ||
|
|
621349a085 | ||
|
|
ea3b5a2e29 | ||
|
|
a6b510c7c5 | ||
|
|
0fb06670d9 | ||
|
|
8bed6c6452 | ||
|
|
bc921cac0e | ||
|
|
9b82535190 | ||
|
|
cb1bc6ab62 | ||
|
|
07e50b72b6 | ||
|
|
af2c54c8f0 | ||
|
|
3f55440752 | ||
|
|
0a27dac9cd | ||
|
|
9e601d4d56 | ||
|
|
14b54cab0c | ||
|
|
f11bd7ba05 | ||
|
|
dd99e3e54c | ||
|
|
0b5f815c7b | ||
|
|
4f2e94c8c5 | ||
|
|
669050d025 | ||
|
|
13f8d7bc23 | ||
|
|
e441d2d6e9 | ||
|
|
179506e8be | ||
|
|
123d05b7d3 | ||
|
|
c76ec2b61a | ||
|
|
20da980cc8 | ||
|
|
a2d8118978 | ||
|
|
0cdd6c543f | ||
|
|
9123cfb1f4 | ||
|
|
1bfbdd2826 | ||
|
|
449fcf394c | ||
|
|
9c56f40170 | ||
|
|
96256ecbf8 | ||
|
|
bdc46b8c6c | ||
|
|
cdb5b70a8d | ||
|
|
66cfb0750d | ||
|
|
4fea42488a | ||
|
|
de8b8189d7 | ||
|
|
dc194c4e54 | ||
|
|
4c3911b5e1 | ||
|
|
93b835e1fd | ||
|
|
c2d8dbd746 | ||
|
|
6cebbb3cd9 | ||
|
|
c32b5703ac | ||
|
|
f859cbfd7a | ||
|
|
431341c235 | ||
|
|
e9847a4424 | ||
|
|
4dc0522cf2 | ||
|
|
6560dfc470 | ||
|
|
2bd0edb264 | ||
|
|
134ebf399e | ||
|
|
1ba76af353 | ||
|
|
a119b60ca6 | ||
|
|
62b3861da0 | ||
|
|
1f729c4764 | ||
|
|
f113b32de0 | ||
|
|
cade7f37aa | ||
|
|
58fc4abca8 | ||
|
|
57bcae38e9 | ||
|
|
3fe38c8c63 | ||
|
|
aa1a0bd38b | ||
|
|
32302e7b17 | ||
|
|
3c338e855a | ||
|
|
98d2596fbf | ||
|
|
f105361658 | ||
|
|
943b5dd9f0 | ||
|
|
c98f828c25 | ||
|
|
ffef55283a | ||
|
|
4455c2cd40 | ||
|
|
81d1644cd7 | ||
|
|
9d03ad0b37 | ||
|
|
59874c717d | ||
|
|
38723c8b81 | ||
|
|
d27f21b10f | ||
|
|
3c3ef82ae8 | ||
|
|
13081f7d07 | ||
|
|
ff5f399e2c | ||
|
|
0025fae492 | ||
|
|
7d70bf0cf4 | ||
|
|
b1c36ad04e | ||
|
|
94a588c7a3 | ||
|
|
b2b5c2ade2 | ||
|
|
ca4cd230a4 | ||
|
|
9c713d43bf | ||
|
|
1a74bdbb6a | ||
|
|
80120aad02 | ||
|
|
cc5fee0956 | ||
|
|
6848fe32fd | ||
|
|
7992470cc3 | ||
|
|
feed82c15d | ||
|
|
ba4fd9d9fd | ||
|
|
ecded4c94f | ||
|
|
c216899c2f | ||
|
|
6faec2b2a9 | ||
|
|
c61381dc79 | ||
|
|
7e4cacd151 | ||
|
|
834f3e5b84 | ||
|
|
1624c77fbd | ||
|
|
9ac181dbf4 | ||
|
|
1d5b1f4f81 | ||
|
|
d34e292e54 | ||
|
|
6c8ddac627 | ||
|
|
b10ae3eabc | ||
|
|
f4d6966404 | ||
|
|
62563f5c80 | ||
|
|
7161c86f73 | ||
|
|
9fceae63fc | ||
|
|
8b82906c7b | ||
|
|
a4c394b827 | ||
|
|
b7979e2f9f | ||
|
|
234869d41e | ||
|
|
f8493a47f1 | ||
|
|
c30b38d220 | ||
|
|
5c3c92da28 | ||
|
|
ee3a6d1776 | ||
|
|
731cdd35fc | ||
|
|
a1a6c6cfac | ||
|
|
dfa0f6861f | ||
|
|
772569cefe | ||
|
|
196f753d1a | ||
|
|
0d651cc947 | ||
|
|
c9294947b1 | ||
|
|
c1f12d1b1b | ||
|
|
731235f6a5 | ||
|
|
d4520677cc | ||
|
|
486ecafbb5 | ||
|
|
518cc7eac7 | ||
|
|
0be713b30c | ||
|
|
fdfa1c16b8 | ||
|
|
485c6e28fd | ||
|
|
5a366ec42b | ||
|
|
e159f0b4ba | ||
|
|
be06b32c82 | ||
|
|
1ab4e1f269 | ||
|
|
3f1254d374 | ||
|
|
9272203d8c | ||
|
|
c19f0d6c6f | ||
|
|
0e73e6c240 | ||
|
|
481a577937 | ||
|
|
b403954ced | ||
|
|
9f62c6977c | ||
|
|
3f49271f63 | ||
|
|
311fbd05dd | ||
|
|
5d6be3ad4b | ||
|
|
fa839b6e37 | ||
|
|
416fba8dab | ||
|
|
edb7051508 | ||
|
|
17d156a7ca | ||
|
|
7406023c93 | ||
|
|
6c071e7248 | ||
|
|
f4ea8a6d75 | ||
|
|
c89d31d0db | ||
|
|
db34df0a35 | ||
|
|
b5e4213e81 | ||
|
|
f4ba5faddc | ||
|
|
24f516077c | ||
|
|
aab0a804b9 | ||
|
|
747eef2c6b | ||
|
|
aff528358e | ||
|
|
f3cd1e6ab5 | ||
|
|
8ff06ed008 | ||
|
|
e1320c8817 | ||
|
|
a6da4a4a98 | ||
|
|
7922224839 | ||
|
|
cc7cd5cd39 | ||
|
|
534f7bb901 | ||
|
|
e074925327 | ||
|
|
f2b6eff29d | ||
|
|
30b9194e43 | ||
|
|
5fbe4d24f6 | ||
|
|
7ce164fba9 | ||
|
|
773148486e | ||
|
|
b26e8b753c | ||
|
|
74e9be6443 | ||
|
|
8fee2370f8 | ||
|
|
c3c806410f | ||
|
|
387a79962f | ||
|
|
d9b57f28e2 | ||
|
|
c99406e360 | ||
|
|
49efc3846c | ||
|
|
690db33147 | ||
|
|
0bb1030ca2 | ||
|
|
1d20deb51f | ||
|
|
590f61a550 | ||
|
|
1054b96e46 | ||
|
|
5bda20740c | ||
|
|
808f265fab | ||
|
|
2637061601 | ||
|
|
0f3f548952 | ||
|
|
e4814dea76 | ||
|
|
87986d88d0 | ||
|
|
f08ae2572d | ||
|
|
d27b177817 | ||
|
|
0b01c2c413 | ||
|
|
bfe73a4dc9 | ||
|
|
01b0926748 | ||
|
|
d9e6059079 | ||
|
|
b4d4ad218a | ||
|
|
3a23798094 | ||
|
|
a3397be489 | ||
|
|
a720467d3b | ||
|
|
cba9e7a65a | ||
|
|
e8dda26b01 | ||
|
|
578100f5ef | ||
|
|
deb8305a1d | ||
|
|
0788e7c211 | ||
|
|
74dd8bc6fc | ||
|
|
3875e36ecf | ||
|
|
7641c016d8 | ||
|
|
8f580a420c | ||
|
|
65fe0a7f51 | ||
|
|
932f3c3908 | ||
|
|
ee54942b01 | ||
|
|
3c28e9b7f0 | ||
|
|
ea13ecb998 | ||
|
|
c152298ad3 | ||
|
|
0b6070c6e1 | ||
|
|
9f48ae2d8b | ||
|
|
353a7fd61d | ||
|
|
2e550c3d35 | ||
|
|
60959f5812 | ||
|
|
ed8b992c80 | ||
|
|
b784f86249 | ||
|
|
d637ae7dbf | ||
|
|
1a72e59254 | ||
|
|
99f10fb2d4 | ||
|
|
5ba7ecdca7 | ||
|
|
c95c40c3fe | ||
|
|
24d48c5e6e | ||
|
|
e2e2b632ff | ||
|
|
2728d2f50a | ||
|
|
f969da014d | ||
|
|
107a0d3c85 | ||
|
|
44feb2f08a | ||
|
|
86afa748de | ||
|
|
ef1c6accb6 | ||
|
|
291e7a382b | ||
|
|
fd39824c0e | ||
|
|
77c8309a4c | ||
|
|
ac1e0ad6a3 | ||
|
|
7926d25c79 | ||
|
|
09b97c0d32 | ||
|
|
bcc28795e4 | ||
|
|
c5e2770bab | ||
|
|
eda8b7938d | ||
|
|
b3cdf438f9 | ||
|
|
4d51608118 | ||
|
|
e97286da01 | ||
|
|
4f0ecbe3c0 | ||
|
|
5fef177e36 | ||
|
|
e1f711dea7 | ||
|
|
08ba09c955 | ||
|
|
f2a6b1081e | ||
|
|
9fc70631a8 | ||
|
|
c205da8b2b | ||
|
|
8f818ac0c9 | ||
|
|
a639cf60de | ||
|
|
b2fae0aa70 | ||
|
|
c3cf485cd9 | ||
|
|
4a5ca0b9eb | ||
|
|
03f97e63d6 | ||
|
|
f73febbd55 | ||
|
|
6ceb75e717 | ||
|
|
bf0aefa050 | ||
|
|
4f46580210 | ||
|
|
77510cd12a | ||
|
|
a75d232ceb | ||
|
|
dae57776fc | ||
|
|
e4485c9eb6 |
@@ -1,6 +1,12 @@
|
||||
# Silo Docker deployment
|
||||
# For the default docker compose setup, MEDIA_ROOT is the main value you need to set.
|
||||
|
||||
# Optional: layer compose files through env. Leave unset for CPU-only defaults.
|
||||
# Linux/macOS example for enabling NVIDIA override:
|
||||
# COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
|
||||
# Windows example:
|
||||
# COMPOSE_FILE=docker-compose.yml;docker-compose.nvidia.yml
|
||||
|
||||
# Required for docker compose: host path to your media library.
|
||||
# Example: MEDIA_ROOT=/srv/media
|
||||
MEDIA_ROOT=/srv/media
|
||||
@@ -20,6 +26,19 @@ SILO_IMAGE=ghcr.io/silo-server/silo-server:latest
|
||||
POSTGRES_USER=silo
|
||||
POSTGRES_PASSWORD=silo
|
||||
POSTGRES_DB=silo
|
||||
POSTGRES_SHM_SIZE=8gb
|
||||
|
||||
# PostgreSQL auto-tuning is enabled by default in docker-compose.yml. Silo
|
||||
# applies pgtune-style OLTP recommendations through ALTER SYSTEM at startup.
|
||||
# Set POSTGRES_TUNE=off if you manage PostgreSQL tuning yourself.
|
||||
# POSTGRES_TUNE=off
|
||||
# POSTGRES_TUNE_PROFILE=oltp
|
||||
# POSTGRES_TUNE_MEMORY=auto
|
||||
# POSTGRES_TUNE_MEMORY_BUDGET_PERCENT=75
|
||||
# POSTGRES_TUNE_CPUS=auto
|
||||
# POSTGRES_TUNE_STORAGE=ssd
|
||||
# POSTGRES_TUNE_DB_SIZE=auto
|
||||
# POSTGRES_TUNE_CONNECTIONS=100
|
||||
|
||||
# Optional host port overrides for docker-compose.yml.
|
||||
# PORT=8090
|
||||
@@ -27,6 +46,19 @@ POSTGRES_DB=silo
|
||||
# PROXY_PORT=8083
|
||||
# TRANSCODE_PORT=8082
|
||||
|
||||
# Optional NVIDIA override controls (used by docker-compose.nvidia.yml).
|
||||
# NVIDIA_GPU_COUNT=1
|
||||
|
||||
# REQUIRED: master key for at-rest credential encryption. Silo refuses to start
|
||||
# without it. Integration API keys, S3 keys, and other server-owned secrets are
|
||||
# encrypted under a key derived from this value, so it must be at least 32
|
||||
# characters and kept secret. Generate one with:
|
||||
# openssl rand -base64 48
|
||||
# Back it up SEPARATELY from your database dumps (treat it like a CA private
|
||||
# key): losing it makes every encrypted secret unrecoverable. See
|
||||
# docs/architecture/secret-encryption.md.
|
||||
# SECRET_KEY=replace-with-output-of-openssl-rand-base64-48
|
||||
|
||||
# Run from source / advanced overrides
|
||||
# Only DATABASE_URL is required when running Silo outside the default docker compose stack.
|
||||
# DATABASE_URL=postgres://silo:password@localhost:5432/silo
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Repository Guidelines
|
||||
|
||||
## Project Structure & Module Organization
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/` as paired numbered `.up.sql` and `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/sql/` as Goose SQL migrations. Legacy converted migrations intentionally keep their original numeric versions so existing `schema_versions` rows can bootstrap cleanly into Goose. New migrations must use timestamped filenames created with `make migrate-create NAME=add_thing`; do not run `goose fix` or create paired `.up.sql` / `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
|
||||
When creating or editing `docs/superpowers/specs/` or `docs/superpowers/plans/`, never include local absolute filesystem paths or transient worktree IDs. Use repository-relative paths and wording like "Commands assume the repository root is the cwd."
|
||||
|
||||
@@ -53,6 +53,7 @@ Use the checked-in `Makefile` for the common paths:
|
||||
- `make dev-frontend`: start the Vite dev server with HMR
|
||||
- `make dev-proxy` / `make dev-transcode`: run standalone worker modes
|
||||
- `make lint`: run `golangci-lint` and frontend ESLint
|
||||
- `make migrate-status` / `make migrate-up`: inspect or apply Goose migrations through Silo's legacy-safe bootstrapping runner
|
||||
|
||||
Run before opening a merge request:
|
||||
|
||||
|
||||
+3
-1
@@ -3,6 +3,7 @@ FROM node:22-slim AS frontend
|
||||
RUN corepack enable && corepack prepare pnpm@10.32.1 --activate
|
||||
WORKDIR /app/web
|
||||
COPY web/package.json web/pnpm-lock.yaml ./
|
||||
COPY web/vendor/foliate-js ./vendor/foliate-js
|
||||
RUN --mount=type=cache,target=/root/.local/share/pnpm/store \
|
||||
pnpm install --frozen-lockfile
|
||||
COPY web/ .
|
||||
@@ -22,6 +23,7 @@ ENV GONOSUMDB=github.com/Silo-Server/*
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libvips-dev && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
COPY internal/compat/zishang520-webtransport-go/ internal/compat/zishang520-webtransport-go/
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/go/pkg/mod \
|
||||
go mod download
|
||||
@@ -48,7 +50,7 @@ RUN apt-get update && \
|
||||
echo "deb [signed-by=/usr/share/keyrings/jellyfin.gpg arch=${TARGETARCH}] https://repo.jellyfin.org/debian bookworm main" \
|
||||
> /etc/apt/sources.list.d/jellyfin.list && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 fonts-noto-core fonts-noto-cjk && \
|
||||
apt-get purge -y gnupg && apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN mkdir -p /tmp/silo-transcode
|
||||
|
||||
+3
-1
@@ -5,6 +5,7 @@ FROM node:22-slim AS frontend
|
||||
RUN corepack enable && corepack prepare pnpm@10.32.1 --activate
|
||||
WORKDIR /app/web
|
||||
COPY web/package.json web/pnpm-lock.yaml ./
|
||||
COPY web/vendor/foliate-js ./vendor/foliate-js
|
||||
RUN --mount=type=cache,target=/root/.local/share/pnpm/store \
|
||||
pnpm install --frozen-lockfile
|
||||
COPY web/ .
|
||||
@@ -19,6 +20,7 @@ ENV GONOSUMDB=github.com/Silo-Server/*
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libvips-dev && rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
COPY internal/compat/zishang520-webtransport-go/ internal/compat/zishang520-webtransport-go/
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/go/pkg/mod \
|
||||
go mod download
|
||||
@@ -53,7 +55,7 @@ RUN apt-get update && \
|
||||
echo "deb [signed-by=/usr/share/keyrings/jellyfin.gpg arch=${TARGETARCH}] https://repo.jellyfin.org/debian bookworm main" \
|
||||
> /etc/apt/sources.list.d/jellyfin.list && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 && \
|
||||
apt-get install -y --no-install-recommends jellyfin-ffmpeg7 libvips42 fonts-noto-core fonts-noto-cjk && \
|
||||
apt-get purge -y gnupg && apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN mkdir -p /tmp/silo-transcode
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks migrate-create migrate-validate migrate-status migrate-up
|
||||
|
||||
GIT_COMMON_DIR := $(strip $(shell git rev-parse --git-common-dir 2>/dev/null))
|
||||
MAIN_CHECKOUT_ROOT := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/..))
|
||||
SHARED_MAKEFILE_LOCAL := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/../Makefile.local))
|
||||
DEFAULT_PLUGIN_SDK_DIR := $(abspath ../silo-plugin-sdk)
|
||||
SHARED_PLUGIN_SDK_DIR := $(if $(MAIN_CHECKOUT_ROOT),$(abspath $(MAIN_CHECKOUT_ROOT)/../silo-plugin-sdk))
|
||||
GOOSE := go run github.com/pressly/goose/v3/cmd/goose@v3.27.1
|
||||
GOOSE_DIR := migrations/sql
|
||||
ENV_FILE ?= .env
|
||||
|
||||
ifneq ($(wildcard $(DEFAULT_PLUGIN_SDK_DIR)),)
|
||||
DEV_PLUGIN_SDK_DIR ?= $(DEFAULT_PLUGIN_SDK_DIR)
|
||||
@@ -14,13 +17,20 @@ endif
|
||||
|
||||
JELLYFIN_WEB_OUTPUT_DIR ?= third_party/jellyfin-web
|
||||
|
||||
# Build version stamping: inject the git revision so the admin Build panel shows a
|
||||
# version even when Go's VCS metadata isn't embedded (mirrors the Dockerfile ldflags).
|
||||
BUILDINFO_PKG := github.com/Silo-Server/silo-server/internal/buildinfo
|
||||
BUILD_REVISION ?= $(shell git rev-parse HEAD 2>/dev/null)
|
||||
BUILD_DIRTY ?= $(shell test -n "$$(git status --porcelain 2>/dev/null)" && echo true || echo false)
|
||||
GO_LDFLAGS := -X $(BUILDINFO_PKG).revisionOverride=$(BUILD_REVISION) -X $(BUILDINFO_PKG).dirtyOverride=$(BUILD_DIRTY)
|
||||
|
||||
# Build the frontend (requires pnpm)
|
||||
frontend:
|
||||
cd web && pnpm install --frozen-lockfile && pnpm run build
|
||||
|
||||
# Build the Go binary (depends on frontend)
|
||||
build: frontend
|
||||
go build -o silo ./cmd/silo/
|
||||
go build -ldflags "$(GO_LDFLAGS)" -o silo ./cmd/silo/
|
||||
|
||||
# Run frontend dev server (proxies API to localhost:8080)
|
||||
dev-frontend:
|
||||
@@ -47,6 +57,23 @@ lint:
|
||||
verify-local-paths:
|
||||
scripts/check-local-path-leaks.sh
|
||||
|
||||
# Create a timestamped Goose SQL migration. Usage: make migrate-create NAME=add_thing
|
||||
migrate-create:
|
||||
@if [ -z "$(NAME)" ]; then echo "usage: make migrate-create NAME=add_thing"; exit 1; fi
|
||||
$(GOOSE) -dir $(GOOSE_DIR) create "$(NAME)" sql
|
||||
|
||||
# Validate Goose migration annotations and SQL parsing without touching a database.
|
||||
migrate-validate:
|
||||
$(GOOSE) -dir $(GOOSE_DIR) validate
|
||||
|
||||
# Show Goose migration status through Silo's bootstrapping runner.
|
||||
migrate-status:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-status
|
||||
|
||||
# Apply pending Goose migrations through Silo's bootstrapping runner.
|
||||
migrate-up:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-only
|
||||
|
||||
# Install repo-local git hooks for this checkout/worktree.
|
||||
install-hooks:
|
||||
@existing="$$(git config --local core.hooksPath 2>/dev/null || true)"; \
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
A self-hosted media streaming server with a React frontend and Go backend. Supports direct play, remuxing, and hardware-accelerated transcoding. Compatible with Jellyfin/Emby clients (VidHub, Findroid) via a built-in compatibility layer.
|
||||
|
||||
Join the community on [Discord](https://discord.gg/4RxuUQAEnW).
|
||||
|
||||
## Deploy with Docker (recommended)
|
||||
|
||||
The easiest way to run Silo is with Docker Compose. The default stack assumes you do not already have PostgreSQL and Redis available, so it bundles PostgreSQL, Redis, FFmpeg, and the application for a one-command start.
|
||||
@@ -32,6 +34,29 @@ The easiest way to run Silo is with Docker Compose. The default stack assumes yo
|
||||
|
||||
If you already have PostgreSQL and Redis available, omit those bundled service examples from compose and point Silo at your existing `DATABASE_URL` and `REDIS_URL` instead.
|
||||
|
||||
### Optional NVIDIA/NVENC
|
||||
|
||||
GPU support is kept out of the default compose file so hosts without NVIDIA drivers work unchanged.
|
||||
|
||||
Install the NVIDIA Container Toolkit and use a Docker Compose version with GPU reservation support before enabling this override.
|
||||
|
||||
Use the optional override file when you want NVENC:
|
||||
|
||||
```sh
|
||||
docker compose -f docker-compose.yml -f docker-compose.nvidia.yml up -d
|
||||
```
|
||||
|
||||
If you want this controlled from `.env`, set `COMPOSE_FILE`:
|
||||
|
||||
```dotenv
|
||||
COMPOSE_FILE=docker-compose.yml:docker-compose.nvidia.yml
|
||||
NVIDIA_GPU_COUNT=1
|
||||
```
|
||||
|
||||
Windows uses `;` instead of `:` between compose files.
|
||||
|
||||
Then `docker compose up -d` will include the NVIDIA override automatically.
|
||||
|
||||
4. **Configure through the admin UI**
|
||||
|
||||
Add libraries, users, metadata providers, and playback settings from the web interface.
|
||||
@@ -176,8 +201,36 @@ Technical notes:
|
||||
| `make dev-backend` | Run Go backend (integrated mode) |
|
||||
| `make dev-proxy` | Run a standalone proxy node |
|
||||
| `make dev-transcode` | Run a standalone transcode node |
|
||||
| `make migrate-create NAME=add_thing` | Create a timestamped Goose SQL migration |
|
||||
| `make migrate-validate` | Validate Goose migration files without touching a database |
|
||||
| `make migrate-status` | Show Goose migration status using Silo's bootstrapping runner |
|
||||
| `make migrate-up` | Apply pending Goose migrations using Silo's bootstrapping runner |
|
||||
| `make clean` | Remove build artifacts |
|
||||
|
||||
### Database Migrations
|
||||
|
||||
PostgreSQL schema migrations are managed by Goose. Migration SQL files live in
|
||||
`migrations/sql/` and use Goose annotations. Converted legacy migrations keep
|
||||
their original numeric versions so existing `schema_versions` rows can bootstrap
|
||||
cleanly into Goose without replaying old SQL. New migrations should be created
|
||||
with timestamped filenames:
|
||||
|
||||
```sh
|
||||
make migrate-create NAME=add_thing
|
||||
make migrate-validate
|
||||
```
|
||||
|
||||
Do not run `goose fix`; timestamped migrations are the repository policy because
|
||||
they avoid version collisions across parallel PRs. The existing `001`-style
|
||||
files are historical compatibility records, not the naming pattern for new work.
|
||||
Runtime migrations are applied by the integrated/API server only. Proxy and
|
||||
transcode modes never mutate schema.
|
||||
For existing installs, use `make migrate-status` and `make migrate-up` rather
|
||||
than invoking the Goose CLI directly; those targets copy legacy
|
||||
`schema_versions` rows into `public.goose_db_version` under the migration lock
|
||||
before reading or applying migrations. Set `ENV_FILE=path/to/.env` when the
|
||||
database URL should be read from a non-default env file.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```sh
|
||||
@@ -216,93 +269,57 @@ Silo requires only a `DATABASE_URL` when running from source or against external
|
||||
| `proxy` | Stream proxy node that connects to the shared deployment database and Redis |
|
||||
| `transcode` | HLS transcode worker node that connects to the shared deployment database and Redis |
|
||||
|
||||
## PostgreSQL Performance Tuning
|
||||
## PostgreSQL Auto-Tuning
|
||||
|
||||
The default PostgreSQL configuration is conservative and not optimized for any particular workload. For better performance, use [PGTune](https://pgtune.leopard.in.ua/) to generate settings tailored to your hardware.
|
||||
|
||||
**Recommended PGTune settings:**
|
||||
|
||||
| Setting | Value |
|
||||
|---|---|
|
||||
| DB Type | **Web application** |
|
||||
| Total Memory | Your server's RAM |
|
||||
| Number of CPUs | Your server's CPU count |
|
||||
| Number of Connections | 20 (Silo's default pool size) |
|
||||
| Data Storage | SSD or HDD depending on your setup |
|
||||
|
||||
Silo's workload is predominantly read-heavy API queries (browse, search, metadata lookups) with the database small enough to fit in RAM for most libraries. The **Web application** profile optimizes for this: CPU-bound simple queries served from cached pages, high concurrent reads, and low write volume.
|
||||
|
||||
If you have a very large library (50,000+ items) or heavy activity logging, consider switching to the **OLTP** profile instead.
|
||||
|
||||
### Recommended Defaults for Modern Hardware
|
||||
|
||||
Assuming you're using modern hardware with fast NVMe storage this is what I'm currently running.
|
||||
|
||||
```ini
|
||||
# Connections
|
||||
max_connections = 100
|
||||
|
||||
# Memory (tuned to keep PG under ~8GB total)
|
||||
shared_buffers = 4GB # holds the entire DB in cache (typical DB is 1-4GB)
|
||||
effective_cache_size = 6GB # planner hint only, not an allocation
|
||||
work_mem = 32MB # per-operation sort/hash memory
|
||||
maintenance_work_mem = 512MB # VACUUM, CREATE INDEX, catalog imports
|
||||
|
||||
# WAL / Write Performance
|
||||
wal_buffers = 64MB
|
||||
min_wal_size = 512MB
|
||||
max_wal_size = 2GB
|
||||
checkpoint_completion_target = 0.9
|
||||
|
||||
# Storage (NVMe/Optane)
|
||||
random_page_cost = 1.1 # default 4.0; NVMe random ≈ sequential
|
||||
effective_io_concurrency = 200 # NVMe handles many concurrent requests
|
||||
|
||||
# Parallelism
|
||||
max_worker_processes = 8
|
||||
max_parallel_workers_per_gather = 4
|
||||
max_parallel_workers = 8
|
||||
max_parallel_maintenance_workers = 4
|
||||
|
||||
# Huge Pages
|
||||
huge_pages = try
|
||||
```
|
||||
|
||||
These settings keep PostgreSQL under ~8GB of RAM while still caching the entire database. For HDD storage, change `random_page_cost` to `4.0` and `effective_io_concurrency` to `2`.
|
||||
|
||||
### Applying Settings
|
||||
|
||||
**Docker (recommended):** Save the settings above to a `postgresql.conf` file and mount it into your container:
|
||||
The default Docker Compose stack does not require a checked-in `postgresql.conf`.
|
||||
It enables Silo's [pgtune](https://github.com/le0pard/pgtune)-style OLTP tuning
|
||||
by default:
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
volumes:
|
||||
- ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
||||
POSTGRES_TUNE: auto
|
||||
```
|
||||
|
||||
Then restart: `docker compose restart postgres`
|
||||
When enabled, Silo connects with `DATABASE_URL` and applies recommendations with
|
||||
`ALTER SYSTEM`, which writes to PostgreSQL's `postgresql.auto.conf` inside the
|
||||
database data directory. Reloadable settings are applied immediately with
|
||||
`pg_reload_conf()`. Settings that PostgreSQL marks as restart-only are written
|
||||
too, and Silo logs the setting names so you can restart PostgreSQL once:
|
||||
|
||||
**Existing PostgreSQL:** You can apply settings without editing config files using `ALTER SYSTEM`, which writes to `postgresql.auto.conf` inside the data directory:
|
||||
|
||||
```sql
|
||||
ALTER SYSTEM SET work_mem = '32MB';
|
||||
ALTER SYSTEM SET effective_cache_size = '6GB';
|
||||
ALTER SYSTEM SET random_page_cost = 1.1;
|
||||
ALTER SYSTEM SET effective_io_concurrency = 200;
|
||||
-- Apply without restart:
|
||||
SELECT pg_reload_conf();
|
||||
|
||||
-- These require a full restart:
|
||||
ALTER SYSTEM SET shared_buffers = '4GB';
|
||||
ALTER SYSTEM SET max_connections = 100;
|
||||
ALTER SYSTEM SET max_worker_processes = 8;
|
||||
ALTER SYSTEM SET huge_pages = 'try';
|
||||
```sh
|
||||
docker compose restart postgres
|
||||
```
|
||||
|
||||
Settings like `shared_buffers`, `max_connections`, `max_worker_processes`, and `huge_pages` require a PostgreSQL restart. Memory/planner settings (`work_mem`, `effective_cache_size`, `random_page_cost`) can be reloaded live.
|
||||
The default Compose database user has the required PostgreSQL permissions. If
|
||||
you use an external PostgreSQL server, make sure the configured `DATABASE_URL`
|
||||
user can run `ALTER SYSTEM`, or set `POSTGRES_TUNE=off` and manage
|
||||
PostgreSQL yourself.
|
||||
|
||||
For `POSTGRES_TUNE_MEMORY=auto`, Silo uses the first trustworthy memory source:
|
||||
a finite Docker cgroup limit, the read-only `/host/proc/meminfo` mount supplied
|
||||
by the bundled Compose file, then `/proc/meminfo` with container safety guards.
|
||||
Auto-detected memory is treated as a PostgreSQL budget, defaulting to 75% of
|
||||
detected RAM so Silo, Redis, plugins, transcodes, and the OS retain headroom.
|
||||
`POSTGRES_TUNE_DB_SIZE=auto` queries `pg_database_size(current_database())` and
|
||||
classifies the workload by comparing the database size to that memory budget.
|
||||
|
||||
Optional tuning overrides:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---:|---|
|
||||
| `POSTGRES_TUNE_PROFILE` | `oltp` | Tuning profile. Only `oltp` is currently supported. |
|
||||
| `POSTGRES_TUNE_MEMORY` | `auto` | Server/container RAM, such as `8GB` or `32GB`; explicit values are used as-is. |
|
||||
| `POSTGRES_TUNE_MEMORY_BUDGET_PERCENT` | `75` | Percent of auto-detected RAM used for PostgreSQL recommendations. |
|
||||
| `POSTGRES_TUNE_CPUS` | `auto` | CPU count used for worker recommendations. |
|
||||
| `POSTGRES_TUNE_STORAGE` | `ssd` | One of `hdd`, `ssd`, `san`, or `nvme`. |
|
||||
| `POSTGRES_TUNE_DB_SIZE` | `auto` | Use `less_ram` when the database comfortably fits in RAM, `mid_ram`, or `greater_ram` for very large databases. |
|
||||
| `POSTGRES_TUNE_CONNECTIONS` | `100` | PostgreSQL `max_connections`; automatically raised if Silo's app pool is configured higher. |
|
||||
| `POSTGRES_SHM_SIZE` | `8gb` | Docker `/dev/shm` size for the bundled PostgreSQL container. |
|
||||
|
||||
Advanced operators can still supply their own PostgreSQL configuration or
|
||||
override these env vars. Set `POSTGRES_TUNE=off` when you do not want Silo to
|
||||
change PostgreSQL server settings. Settings already written with `ALTER SYSTEM`
|
||||
remain in `postgresql.auto.conf`; reset those PostgreSQL parameters if you later
|
||||
move fully to a custom `postgresql.conf`.
|
||||
|
||||
## Project Structure
|
||||
|
||||
@@ -319,5 +336,5 @@ internal/
|
||||
scanner/ Media file discovery and FFProbe
|
||||
worker/ Background jobs (scan, match, reconcile)
|
||||
web/ React + TypeScript frontend (Vite, Tailwind, shadcn/ui)
|
||||
migrations/ PostgreSQL schema migrations
|
||||
migrations/sql/ Goose-managed PostgreSQL schema migrations
|
||||
```
|
||||
|
||||
+691
-129
File diff suppressed because it is too large
Load Diff
@@ -132,6 +132,9 @@ func TestBuildLiveSessionSync_UsesTransportPlayMethod(t *testing.T) {
|
||||
RequestedMediaFileID: 41,
|
||||
PlayMethod: playback.PlayTranscode,
|
||||
BasePlayMethod: playback.PlayRemux,
|
||||
TranscodeHWAccel: "qsv",
|
||||
Position: 125.5,
|
||||
IsPaused: true,
|
||||
},
|
||||
want: "transcode",
|
||||
},
|
||||
@@ -164,6 +167,18 @@ func TestBuildLiveSessionSync_UsesTransportPlayMethod(t *testing.T) {
|
||||
if got.SessionID != tc.session.ID {
|
||||
t.Fatalf("SessionID = %q, want %q", got.SessionID, tc.session.ID)
|
||||
}
|
||||
if got.ProfileID != tc.session.ProfileID {
|
||||
t.Fatalf("ProfileID = %q, want %q", got.ProfileID, tc.session.ProfileID)
|
||||
}
|
||||
if got.PositionSeconds != tc.session.Position {
|
||||
t.Fatalf("PositionSeconds = %v, want %v", got.PositionSeconds, tc.session.Position)
|
||||
}
|
||||
if got.IsPaused != tc.session.IsPaused {
|
||||
t.Fatalf("IsPaused = %v, want %v", got.IsPaused, tc.session.IsPaused)
|
||||
}
|
||||
if got.TranscodeHWAccel != tc.session.TranscodeHWAccel {
|
||||
t.Fatalf("TranscodeHWAccel = %q, want %q", got.TranscodeHWAccel, tc.session.TranscodeHWAccel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
)
|
||||
|
||||
type fakeMarkerRuntimeConfigs struct {
|
||||
configs map[int][]*plugins.RuntimeConfig
|
||||
puts []plugins.RuntimeConfig
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) ListGlobalConfigs(_ context.Context, installationID int) ([]*plugins.RuntimeConfig, error) {
|
||||
return append([]*plugins.RuntimeConfig(nil), f.configs[installationID]...), nil
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) PutGlobalConfig(_ context.Context, installationID int, key string, value map[string]any) error {
|
||||
f.puts = append(f.puts, plugins.RuntimeConfig{InstallationID: installationID, Key: key, Value: value})
|
||||
f.configs[installationID] = append(f.configs[installationID], &plugins.RuntimeConfig{
|
||||
InstallationID: installationID,
|
||||
Key: key,
|
||||
Value: value,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeMarkerLegacySettings map[string]string
|
||||
|
||||
func (f fakeMarkerLegacySettings) Get(_ context.Context, key string) (string, error) {
|
||||
return f[key], nil
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigCopiesAPIKeyOnce(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
installation := &plugins.Installation{ID: 42, PluginID: "silo.theintrodb"}
|
||||
capability := &plugins.Capability{ID: "introdb"}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": " legacy-key "},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
got := runtimeConfigs.puts[0]
|
||||
if got.InstallationID != 42 || got.Key != "account" || got.Value["api_key"] != "legacy-key" {
|
||||
t.Fatalf("put = %+v, want account api_key copy", got)
|
||||
}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "new-key"},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("second copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("second copy overwrote config; puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigIgnoresOtherPlugins(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "legacy-key"},
|
||||
&plugins.Installation{ID: 7, PluginID: "silo.other"},
|
||||
&plugins.Capability{ID: "introdb"},
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 0 {
|
||||
t.Fatalf("puts = %d, want 0 for non-TheIntroDB plugin", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
@@ -31,8 +31,10 @@ func buildLiveSessionSync(s *playback.Session, reportingNode string) worker.Sess
|
||||
TargetVideoCodec: s.TargetVideoCodec,
|
||||
TargetAudioCodec: s.TargetAudioCodec,
|
||||
TargetBitrateKbps: s.TargetBitrateKbps,
|
||||
TranscodeHWAccel: s.TranscodeHWAccel,
|
||||
StartedAt: s.StartedAt,
|
||||
UpdatedAt: s.UpdatedAt,
|
||||
PositionSeconds: s.Position,
|
||||
IsPaused: s.IsPaused,
|
||||
HasWebSocket: s.HasWebSocket,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
services:
|
||||
silo:
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: ${NVIDIA_GPU_COUNT:-1}
|
||||
capabilities: [gpu]
|
||||
|
||||
# Optional distributed-mode example. Most single-host installs should
|
||||
# just use the integrated service above.
|
||||
# silo-transcode:
|
||||
# deploy:
|
||||
# resources:
|
||||
# reservations:
|
||||
# devices:
|
||||
# - driver: nvidia
|
||||
# count: ${NVIDIA_GPU_COUNT:-1}
|
||||
# capabilities: [gpu]
|
||||
+16
-4
@@ -2,7 +2,7 @@ services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
restart: unless-stopped
|
||||
shm_size: 1gb
|
||||
shm_size: ${POSTGRES_SHM_SIZE:-8gb}
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-silo}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-silo}
|
||||
@@ -11,8 +11,7 @@ services:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/postgres:/var/lib/postgresql
|
||||
- ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
||||
command: ["postgres", "-c", "listen_addresses=*"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-silo}"]
|
||||
interval: 5s
|
||||
@@ -37,20 +36,27 @@ services:
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MODE: integrated
|
||||
# Master key for at-rest credential encryption. Required — the server
|
||||
# refuses to start without it. Generate with: openssl rand -base64 48
|
||||
# Back it up SEPARATELY from your database dumps; losing it makes encrypted
|
||||
# secrets unrecoverable. See docs/architecture/secret-encryption.md.
|
||||
SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env — generate one with openssl rand -base64 48}
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
REDIS_URL: redis://redis:6379
|
||||
SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins
|
||||
POSTGRES_TUNE: ${POSTGRES_TUNE:-auto}
|
||||
ports:
|
||||
- "${PORT:-8090}:8080"
|
||||
- "${JF_PORT:-8096}:8096"
|
||||
- "${ABS_PORT:-13378}:13378"
|
||||
volumes:
|
||||
- ${MEDIA_ROOT:?Set MEDIA_ROOT in .env to the host media path}:${MEDIA_CONTAINER_ROOT:-/mnt/media}:ro
|
||||
- ${MEDIA_BOOKS_ROOT:-${MEDIA_ROOT}}:${MEDIA_BOOKS_CONTAINER_ROOT:-${MEDIA_CONTAINER_ROOT:-/mnt/media}}:ro
|
||||
- ${MEDIA_BOOKS_ROOT:-${MEDIA_ROOT}}:${MEDIA_BOOKS_CONTAINER_ROOT:-${MEDIA_CONTAINER_ROOT:-/mnt/media}/books}:ro
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/plugins:/var/lib/silo/plugins
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/transcode:/tmp/silo-transcode
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/catalog-seeds:/catalog-seeds:ro
|
||||
- ${SILO_DATA_ROOT:-/opt/silo}/audiobook-covers:/var/lib/silo/audiobook-covers
|
||||
- /proc/meminfo:/host/proc/meminfo:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -59,6 +65,8 @@ services:
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
|
||||
# Optional distributed-mode example. Most single-host installs should
|
||||
# just use the integrated service above.
|
||||
@@ -68,6 +76,8 @@ services:
|
||||
# profiles: [proxy]
|
||||
# environment:
|
||||
# MODE: proxy
|
||||
# # Must be the SAME SECRET_KEY as the primary node (shared encrypted data).
|
||||
# SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env — generate one with openssl rand -base64 48}
|
||||
# DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
# REDIS_URL: redis://redis:6379
|
||||
# PORT: "8080"
|
||||
@@ -92,6 +102,8 @@ services:
|
||||
# profiles: [transcode]
|
||||
# environment:
|
||||
# MODE: transcode
|
||||
# # Must be the SAME SECRET_KEY as the primary node (shared encrypted data).
|
||||
# SECRET_KEY: ${SECRET_KEY:?Set SECRET_KEY in .env — generate one with openssl rand -base64 48}
|
||||
# DATABASE_URL: postgres://${POSTGRES_USER:-silo}:${POSTGRES_PASSWORD:-silo}@postgres:5432/${POSTGRES_DB:-silo}?sslmode=disable
|
||||
# REDIS_URL: redis://redis:6379
|
||||
# SILO_PLUGIN_CACHE_DIR: /var/lib/silo/plugins
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# At-rest credential encryption
|
||||
|
||||
Silo encrypts third-party integration credentials and other server-owned secrets
|
||||
at rest. Sonarr/Radarr API keys, S3 keys, watch-sync OAuth tokens, the
|
||||
Audiobookshelf-compat signing key, and the sensitive `server_settings` entries
|
||||
are stored as ciphertext in PostgreSQL and decrypted only in memory. The key that
|
||||
protects them lives **outside** the database, so a full DB dump or compromise
|
||||
does not expose the secrets.
|
||||
|
||||
## How it works
|
||||
|
||||
- **Cipher:** AES-256-GCM with a random 12-byte nonce per value. Implemented in
|
||||
`internal/secret`.
|
||||
- **Key derivation:** the data key is HKDF-SHA256–derived from the `SECRET_KEY`
|
||||
environment variable with a versioned domain label. `SECRET_KEY` is the
|
||||
encryption root; it is never stored in the database.
|
||||
- **Envelope:** every ciphertext is stored as `enc:v1:<base64url(nonce‖sealed)>`.
|
||||
The `enc:v1:` prefix is the version marker (see *Key rotation* below).
|
||||
- **Row binding (AAD):** each ciphertext is GCM-bound to its logical row via
|
||||
additional-authenticated data — `table:column:<pk>` for per-row columns,
|
||||
`server_settings:<key>` for settings. A DB-write attacker therefore cannot move
|
||||
a credential blob from one row/column to another: decrypting under a different
|
||||
binding fails authentication.
|
||||
- **Read path:** an empty value stays empty; a non-`enc:v1:` value is treated as
|
||||
legacy plaintext and passed through unchanged (so a not-yet-migrated row keeps
|
||||
working); an `enc:v1:` value is decrypted and **any** failure (wrong key,
|
||||
tampering, truncation) is surfaced as an error — it is never silently used as a
|
||||
credential.
|
||||
|
||||
## SECRET_KEY
|
||||
|
||||
`SECRET_KEY` is **required**. The server refuses to start without it (it fatals
|
||||
in `config.LoadBootstrap`), and it must be at least 32 characters.
|
||||
|
||||
Generate one with:
|
||||
|
||||
```bash
|
||||
openssl rand -base64 48
|
||||
```
|
||||
|
||||
Set it in the deployment environment (or in `.env` for source/dev runs, which
|
||||
`godotenv` loads at startup). See `.env.example`.
|
||||
|
||||
### Back it up — separately from the database
|
||||
|
||||
Treat `SECRET_KEY` like a CA private key:
|
||||
|
||||
- **Store it outside your database backups.** A DB dump contains only ciphertext;
|
||||
the value of at-rest encryption is entirely lost if the key travels with the
|
||||
dump.
|
||||
- Keep it in a secrets manager / sealed secret / password manager, not in the
|
||||
repo or in the same bucket as your Postgres backups.
|
||||
- Rotating the *deployment* (new host, restored DB) requires the **same**
|
||||
`SECRET_KEY`. Restoring a database backup onto a node with a different
|
||||
`SECRET_KEY` makes every encrypted secret unreadable.
|
||||
|
||||
### Key loss
|
||||
|
||||
If `SECRET_KEY` is lost, the encrypted secrets are **unrecoverable** — that is the
|
||||
point of keeping the key out of the database. Recovery means re-entering the
|
||||
affected credentials:
|
||||
|
||||
- Re-enter Sonarr/Radarr and Autoscan API keys, S3 keys, watch-sync connections,
|
||||
history-import tokens, and subtitle provider credentials.
|
||||
- `auth.jwt_secret` becomes unreadable, so all existing sessions are invalid and
|
||||
users must log in again (a new secret is generated on next boot if the row is
|
||||
cleared — see below).
|
||||
|
||||
## Rollout (automatic backfill)
|
||||
|
||||
On the first boot after deploying this change, the primary (migration-running)
|
||||
node runs an idempotent, best-effort startup backfill that encrypts any
|
||||
remaining plaintext in place:
|
||||
|
||||
1. sensitive `server_settings` keys,
|
||||
2. the per-table credential columns (subtitles, watch-sync, webhook-sync,
|
||||
history-import, including temporary server-list credentials stored in
|
||||
session JSON),
|
||||
3. the two arr `api_key_ref` columns — these are **resolved-then-encrypted**: a
|
||||
legacy row that held a `server_settings` reference (e.g.
|
||||
`requests.radarr.api_key`) is collapsed to the real credential before being
|
||||
encrypted.
|
||||
|
||||
The backfill is safe to run repeatedly: already-encrypted values are skipped, and
|
||||
a per-row guard makes concurrent multi-node boots converge without
|
||||
double-encrypting. A failed row is logged and left as plaintext (no new exposure)
|
||||
rather than blocking boot. Proxy/transcode nodes skip the backfill; they read
|
||||
whatever the primary encrypted. No manual steps are required.
|
||||
|
||||
## Rollback / downgrade hazard
|
||||
|
||||
Downgrading to a binary that predates this change is **not** safe while secrets
|
||||
are encrypted, because the old binary has no read path: it would read
|
||||
`enc:v1:auth.jwt_secret` as a literal JWT secret (invalidating all sessions) and
|
||||
read `enc:v1:`-prefixed integration keys as garbage credentials.
|
||||
|
||||
To downgrade safely you must first return the affected values to plaintext, for
|
||||
example:
|
||||
|
||||
- Decrypt the sensitive `server_settings` back to plaintext (run a one-off that
|
||||
reads each via the encrypting repo and writes the plaintext via the raw repo),
|
||||
**or**
|
||||
- At minimum, clear `auth.jwt_secret` so the older binary regenerates a fresh
|
||||
plaintext one (this still logs everyone out), and re-enter any integration
|
||||
credentials as plaintext.
|
||||
|
||||
There is no automatic "decrypt everything" downgrade path in this release; plan
|
||||
downgrades accordingly.
|
||||
|
||||
## Key rotation (future)
|
||||
|
||||
The `enc:v1:` envelope and the HKDF domain label are versioned. A future rotation
|
||||
would introduce `enc:v2:` with a new derived key (and `Decrypt` already dispatches
|
||||
on the version, returning an explicit error for an unknown version). Rotation is
|
||||
out of scope for this release; the versioning exists so it can be added without a
|
||||
data migration of the envelope format.
|
||||
|
||||
## Scope and known gaps
|
||||
|
||||
Covered: arr (Requests + Autoscan) API keys, S3 keys, all sensitive
|
||||
`server_settings`, watch-sync tokens, webhook-sync `access_token`,
|
||||
history-import admin/session tokens and temporary server-list credentials,
|
||||
subtitle provider `api_key`/`password`,
|
||||
the Jellyfin-compat session's bridged Silo access/refresh tokens
|
||||
(`jellycompat_sessions.streamapp_access_token` / `streamapp_refresh_token`), and
|
||||
the ABS signing key.
|
||||
|
||||
Deliberately **not** encrypted (tracked as follow-ups):
|
||||
|
||||
- **Equality-looked-up secrets** — these are matched by exact value
|
||||
(`WHERE … = $1`), and AES-GCM is randomized, so encrypting them would break the
|
||||
lookup. They need a deterministic **blind-index hash** column instead:
|
||||
`api_keys.api_key`, `webhook_sync_connections.webhook_secret`,
|
||||
`jellycompat_sessions.token`, and `watch_together_rooms.join_token`.
|
||||
- **Plugin runtime config** — `plugin_runtime_configs.config_value` is opaque
|
||||
plugin-defined JSONB whose secret fields are manifest-declared and whose runtime
|
||||
lives in a separate repo; encrypting it correctly needs a coordinated,
|
||||
manifest-aware design.
|
||||
|
||||
Excluded (not a gap): `plex_sync_connections.*` is a dead table (zero Go
|
||||
references); `oauth_completion.token_ciphertext` is already AES-GCM;
|
||||
`users.password_hash` and the `*_hash` columns are already hashed.
|
||||
@@ -0,0 +1,98 @@
|
||||
# Wholphin → jellycompat Endpoint Coverage Report
|
||||
|
||||
Date: 2026-06-09
|
||||
Scope: every Jellyfin server endpoint the Wholphin Android TV client (jellyfin-sdk-kotlin based) can call, cross-referenced against the routes registered in `internal/jellycompat/router.go`. Wholphin source analyzed from a local clone of `damontecres/Wholphin` (`app/src/main`). Commands assume the repository root is the cwd.
|
||||
|
||||
Wholphin's Seerr integration (`/api/v1/...` discover/request endpoints) targets a separate Seerr server, not Jellyfin, and is out of scope here.
|
||||
|
||||
## Methodology
|
||||
|
||||
1. Grepped Wholphin for every `*Api.method()` SDK call (`itemsApi`, `playStateApi`, etc.), websocket subscriptions, and raw URL construction; mapped each to its HTTP endpoint per the Jellyfin OpenAPI spec.
|
||||
2. Inventoried every route registered in `internal/jellycompat/router.go`, including path normalization (`path_normalization.go` handles casing and `/emby`//`jellyfin` prefixes).
|
||||
3. For each endpoint Wholphin uses but silo does not serve, checked whether a server-side gate (user policy flag, DTO field) prevents the client from ever calling it.
|
||||
|
||||
## Summary
|
||||
|
||||
- **Covered (full):** 22 endpoint families — everything on the core path: auth, browse, item detail, episodes/next-up, resume, playback info, direct/HLS streaming, subtitles, progress reporting, favorites/played, images, genres, persons, suggestions, media segments (skip intro), display preferences.
|
||||
- **Covered (partial/stub):** 4 — websocket (keep-alive only, no remote control), `/Studios`, `/Users/Public`, `/QuickConnect/Enabled` (deliberate `false`).
|
||||
- **Missing but safely gated:** 8 — the client never calls them because we don't advertise the capability (delete, subtitle download, trickplay, local trailers, additional parts, quick connect auth, live TV, playlists).
|
||||
- **Missing and reachable:** 4 — theme songs, universal audio stream, item download, client log upload. These return 404 when the client calls them. None block video playback.
|
||||
|
||||
**Bottom line: the entire core video path is covered.** The reachable gaps are quality-of-life features (theme songs, photo screensaver, debug log upload) plus one advertised-but-broken inconsistency (`CanDownload=true` with no `/Items/{id}/Download` route).
|
||||
|
||||
## Coverage matrix
|
||||
|
||||
### Fully covered
|
||||
|
||||
| Wholphin call | Endpoint | silo handler |
|
||||
|---|---|---|
|
||||
| `userApi.authenticateUserByName` | `POST /Users/AuthenticateByName` | `handlers_auth.go` |
|
||||
| `userApi.getCurrentUser` | `GET /Users/Me` | `handlers_auth.go` |
|
||||
| `userApi.getUserById` | `GET /Users/{id}` | `handlers_auth.go` |
|
||||
| `systemApi.getPublicSystemInfo` | `GET /System/Info/Public` | `handlers_system.go` |
|
||||
| `sessionApi.postCapabilities` | `POST /Sessions/Capabilities` | `handlers_playback.go` |
|
||||
| `itemsApi.getItems` | `GET /Items` | `handlers_items.go` |
|
||||
| `itemsApi.getResumeItems` | `GET /UserItems/Resume` (+ legacy `/Users/{id}/Items/Resume`) | `handlers_items.go` |
|
||||
| `userLibraryApi.getItem` | `GET /Items/{id}` (+ legacy user-scoped form) | `handlers_items.go` |
|
||||
| `tvShowsApi.getEpisodes` | `GET /Shows/{id}/Episodes` | `handlers_items.go` |
|
||||
| `tvShowsApi.getNextUp` | `GET /Shows/NextUp` | `handlers_items.go` |
|
||||
| `suggestionsApi.getSuggestions` | `GET /Items/Suggestions` | `handlers_items.go` |
|
||||
| `genresApi.getGenres` | `GET /Genres` | `handlers_items.go` |
|
||||
| `personsApi.getPersons` | `GET /Persons` | `handlers_persons.go` (full when PersonRepo is wired; empty stub otherwise) |
|
||||
| `searchApi.getSearchHints` | `GET /Search/Hints` | `handlers_items.go` (Wholphin's call is currently commented out upstream) |
|
||||
| `mediaInfoApi.getPostedPlaybackInfo` | `POST /Items/{id}/PlaybackInfo` | `handlers_playback.go` |
|
||||
| `videosApi.getVideoStreamUrl` | `GET/HEAD /Videos/{id}/stream(.{container})` | `streams.go` |
|
||||
| (transcode path from PlaybackInfo) | `GET /Videos/{id}/master.m3u8` + HLS playlist/segments | `streams.go` |
|
||||
| (subtitle delivery URLs) | `GET /Videos/{id}/{source}/Subtitles/{idx}/stream.{fmt}` | `streams.go` |
|
||||
| `playStateApi.reportPlaybackStart/Progress/Stopped` | `POST /Sessions/Playing[/Progress|/Stopped]` | `streams.go` |
|
||||
| `playStateApi.markPlayedItem/markUnplayedItem` | `POST/DELETE /UserPlayedItems/{id}` (+ legacy) | `handlers_userdata.go` |
|
||||
| `userLibraryApi.markFavoriteItem/unmarkFavoriteItem` | `POST/DELETE /UserFavoriteItems/{id}` (+ legacy) | `handlers_userdata.go` |
|
||||
| `mediaSegmentsApi.getItemSegments` | `GET /MediaSegments/{id}` | `handlers_items.go` (skip intro/credits) |
|
||||
| `displayPreferencesApi.updateDisplayPreferences` | `GET/POST /DisplayPreferences/{id}` | `handlers_displayprefs.go` |
|
||||
| `imageApi.getItemImageUrl` | `GET /Items/{id}/Images/{type}[/{index}]` | `handlers_images.go` |
|
||||
| `imageApi.getUserImageUrl` | `GET /Users/{id}/Images/Primary` | `handlers_images.go` (placeholder avatar) |
|
||||
|
||||
### Partial / stub (graceful degradation)
|
||||
|
||||
| Endpoint | Status | Client impact |
|
||||
|---|---|---|
|
||||
| `GET /socket` (websocket) | Accepts upgrade, answers KeepAlive only (`handlers_websocket.go`) | Wholphin subscribes to `GeneralCommandMessage` and `PlaystateMessage` for server-initiated remote control. Connection stays healthy; remote-control commands are simply never delivered. No errors, feature inert. |
|
||||
| `GET /Studios` | Empty-array stub | Studio browse rows render empty. |
|
||||
| `GET /Users/Public` | Empty-array stub | No user tiles on the login screen; manual login works. |
|
||||
| `GET /QuickConnect/Enabled` | Hardcoded `false` | Deliberate: Wholphin hides Quick Connect UI, so the unimplemented `POST /QuickConnect/Authorize` and `POST /Users/AuthenticateWithQuickConnect` are never called. |
|
||||
|
||||
### Missing but safely gated (client never calls them)
|
||||
|
||||
| Endpoint | Why it's unreachable |
|
||||
|---|---|
|
||||
| `DELETE /Items/{id}` | We return `CanDelete=false` per item (`mapping.go`) and `EnableContentDeletion=false` in user policy (`handlers_auth.go`), so Wholphin hides delete UI. |
|
||||
| `POST /Items/{id}/RemoteSearch/Subtitles/{subtitleId}` | `EnableSubtitleManagement=false` in user policy hides subtitle search/download. |
|
||||
| `GET /Videos/{id}/Trickplay/...` | We never emit trickplay metadata on item DTOs, so the scrubber never requests tiles. |
|
||||
| `GET /Items/{id}/LocalTrailers` | Wholphin gates on `localTrailerCount > 0` (`TrailerService.kt`); we never set `LocalTrailerCount`. |
|
||||
| `GET /Videos/{id}/AdditionalParts` | Gated on `partCount > 1` (`PlaylistCreator.kt`); we never set `PartCount`. |
|
||||
| Quick Connect auth endpoints | Gated by `/QuickConnect/Enabled` = false. |
|
||||
| Live TV suite (`/LiveTv/Channels`, `/Programs`, `/Recordings`, timers) | We expose no live-TV views or channels, so the feature never activates. |
|
||||
| Playlists (`GET/POST/DELETE /Playlists/{id}/Items`) | We expose no playlist items in the catalog, so playlist screens are unreachable. Becomes a real gap the day silo grows playlists. |
|
||||
|
||||
### Missing and reachable (return 404 today)
|
||||
|
||||
| Endpoint | Wholphin caller | Impact | Severity |
|
||||
|---|---|---|---|
|
||||
| `GET /Items/{id}/ThemeSongs` | `ThemeSongPlayer.kt` | ~~404 → feature silently dead.~~ **Resolved 2026-06-09:** now stubbed with an empty `ThemeMediaResult` (including the `OwnerId` field jellyfin-sdk-kotlin requires). | Resolved |
|
||||
| `GET /Audio/{id}/universal` | `ThemeSongPlayer.kt`, `MusicService.kt` | Audio streaming for theme songs and music playback. No music libraries are exposed today, but this is the second half of the theme-song path and the blocker for any future audio support in jellycompat. | Low (today) |
|
||||
| `GET /Items/{id}/Download` | `SlideshowViewModel.kt`, `ScreensaverService.kt` | ~~Inconsistency: `CanDownload=true` while the route 404s.~~ **Resolved 2026-06-09:** a real `/Items/{id}/Download` route now serves the original file (`streams.go`), and `CanDownload=true` again. The earlier `CanDownload=false` workaround broke Infuse, which requires the flag for Direct Play. | Resolved |
|
||||
| `POST /ClientLog/Document` | `MediaReportService.kt`, `DebugPage.kt` | "Upload logs to server" debug action fails. | Low |
|
||||
| `GET /Audio/{id}/Lyrics` | `NowPlayingViewModel.kt` | Music-only; unreachable until audio libraries exist. | Low |
|
||||
|
||||
## Recommendations (priority order)
|
||||
|
||||
1. **Merge the repeated-`Fields` fix** (PR #110). `parseItemsQuery` still reads `q.Get("Fields")` (`internal/jellycompat/query.go`), which truncates jellyfin-sdk-kotlin's repeated query params and breaks Wholphin episode auto-advance. This is the only *covered* endpoint with a known correctness bug for this client. Follow up by sweeping the remaining single-value reads (`Ids`, `GenreIds`, `PersonIds`, `Filters`, `SortBy`, `SortOrder`) to `q.Values`, matching how `IncludeItemTypes`/`MediaTypes`/`ImageTypes` are already handled.
|
||||
2. ~~Resolve the `CanDownload` inconsistency~~ — **done 2026-06-09**: real `/Items/{id}/Download` route added; `CanDownload` stays `true` (Infuse requires it for Direct Play).
|
||||
3. ~~Stub `GET /Items/{id}/ThemeSongs`~~ — **done 2026-06-09**: empty `ThemeMediaResult` stub registered.
|
||||
4. **Websocket server-push** (remote control: server-initiated pause/stop, display messages) — tracked in issue #122.
|
||||
5. **Optional, later:** `POST /ClientLog/Document` accept-and-discard stub and real `/Studios` data. Playlists, lyrics, and universal audio only matter once silo exposes those content types through jellycompat.
|
||||
|
||||
## Notes
|
||||
|
||||
- Path normalization (`path_normalization.go`) already handles Wholphin's casing and any `/emby`//`jellyfin` prefixes, so none of the covered routes are at risk from URL-shape differences.
|
||||
- Unmatched jellycompat paths fall through to chi's default 404 with no body; jellyfin-sdk-kotlin surfaces these as `InvalidStatusException`, which Wholphin generally catches per-feature.
|
||||
@@ -0,0 +1,82 @@
|
||||
# PR #124 (ebooks) — Review & Hardening Pass
|
||||
|
||||
Date: 2026-06-10
|
||||
Scope: full review of the ebook feature branch (`work/ebooks-reader-base`) followed by
|
||||
fixes for every finding. Two review rounds were performed (a full-PR review, then an
|
||||
adversarial edge-case/security pass over both the original code and the first round of
|
||||
fixes), with all changes verified against the full Go test suite, the full web test
|
||||
suite, production frontend build, ESLint/Prettier, and `make verify-local-paths`.
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Security fixes
|
||||
|
||||
- **Stored XSS via book content (high):** book sections render in same-origin blob
|
||||
iframes with `allow-scripts` (WebKit requirement). A Content-Security-Policy is now
|
||||
served with all SPA HTML responses (`internal/server/frontend.go`) — blob/srcdoc
|
||||
documents inherit it, so `script-src 'self' 'wasm-unsafe-eval'` blocks script
|
||||
execution from book content on all browsers. Threat model documented on the policy
|
||||
constant. `X-Content-Type-Options: nosniff` added to frontend, jellycompat, and
|
||||
ebook file responses.
|
||||
- Ebook file serving can no longer fall through to `application/octet-stream` for an
|
||||
admitted file (extension/container whitelist and MIME resolution share one resolver),
|
||||
closing a browser-sniff escape combined with the `?token=` query-auth fallback.
|
||||
- External links in books are intercepted app-side: http(s) only, opened with
|
||||
`noopener,noreferrer` (blocks reverse tabnabbing and `javascript:` URLs).
|
||||
- Request size caps (`http.MaxBytesReader`, 413) on progress/config/annotation writes;
|
||||
`Content-Disposition` built with `mime.FormatMediaType` (RFC 5987, no header
|
||||
injection); annotation PATCH is atomic (`SELECT ... FOR UPDATE`) with
|
||||
presence-aware field semantics and invariant re-validation.
|
||||
- Verified safe under adversarial review: no IDOR (all reader-state SQL scoped to
|
||||
user+profile+content), no path traversal, range serving via stdlib, progress input
|
||||
validation. Accepted residual: CSS-only loads inside book iframes can leak the
|
||||
book's own content but cannot reach tokens, DOM, or navigation.
|
||||
|
||||
## Reliability fixes
|
||||
|
||||
- **Scanner missing-file reconciliation (critical class):** ebook scans now mark
|
||||
missing files like video/audio, with real walk-failure tracking (failed roots are
|
||||
excluded from deletion), symlinked-root support via the shared logical walker, and
|
||||
the empty-root cleanup allowance — an unmounted share can no longer wipe a library.
|
||||
- **Enrichment:** provider errors record failures (dedicated `ebook_enrichment_state`
|
||||
backoff table, capped retries) instead of permanently stamping items done;
|
||||
unconfigured chains and scan-window races skip without stamping. No longer shares
|
||||
`media_items.refresh_failures` with the metadata refresh-debt system.
|
||||
- Ebook items are created `pending` and promoted to `matched` on enrichment (backfill
|
||||
migration included), making curated-metadata protection real: matched items keep
|
||||
provider titles/people/series on re-scan (fill-empty only).
|
||||
- PDF metadata: head+tail window scan (non-linearized PDFs keep the Info dict at the
|
||||
end), delimiter-aware key matching, head-wins merge. `.md` dropped as an ebook
|
||||
format; plain `.fb2` reads capped like `.fbz`.
|
||||
- FK-cascade indexes added for the reader-state tables.
|
||||
|
||||
## Consistency & correctness fixes
|
||||
|
||||
- Hidden-history (`user_history_hidden_items`) gating applied to every ebook progress
|
||||
surface (watched/in-progress filters, all sort plans, Continue Reading, sort
|
||||
metrics, `Played` state, recommendation signals) with the exact video semantics.
|
||||
- `GetItemWatchers` counts distinct watchers (no binge inflation) and the
|
||||
`minWatchers` floor counts distinct accounts, not profiles; ebook reading now feeds
|
||||
implicit taste signals and taste-seed candidates; Continue Reading pages past
|
||||
dismissals and dedupes across pages.
|
||||
- The 0.9 finished threshold is centralized (`models.EbookFinishedProgressThreshold`).
|
||||
- Reader frontend: open-flow race teardown (no wrong-file progress saves), ordered
|
||||
progress writes with `pagehide`/`visibilitychange` flush, settings no longer
|
||||
clobbered by late server config, TTS stop actually stops, Media Session cleanup,
|
||||
512 MiB download guard, fraction bookmarks navigable.
|
||||
|
||||
## Feature completion
|
||||
|
||||
- Native read-state endpoints: `POST/DELETE /watched/{id}` and `/history/remove`
|
||||
accept ebook content IDs (mark read = progress 1.0 preserving location; mark unread
|
||||
mirrors video unwatch and clears position; history removal hides without losing
|
||||
position). Web UI exposes Mark Read/Unread on the item page and card menus, with
|
||||
Continue Reading dismiss copy and dismissal-path ID encoding fixed.
|
||||
|
||||
## Client follow-ups (Android / Apple)
|
||||
|
||||
- Ebook leaf user data encodes the reading ratio as `position_seconds` (0..1) with
|
||||
`duration_seconds = 1`; do not render absolute times for ebooks.
|
||||
- `/watched/{id}` returns `{type: "ebook", affected_count: 1, played: bool}` and the
|
||||
existing watched SSE event fires.
|
||||
- Reading progress is deliberately per-book (cross-format): one position per content
|
||||
ID.
|
||||
@@ -0,0 +1,100 @@
|
||||
# Notifications & Push — Test Checklist
|
||||
|
||||
A repeatable regression pass for the core notifications + push subsystems. Organized by layer: automated (fast, run anytime), server pipeline (API, no browser), real browser (human-in-the-loop), and not-yet-testable (future work). Run the automated layer on every change; run the browser layer before shipping a user-facing change.
|
||||
|
||||
Specs/plans this exercises live under `docs/superpowers/specs|plans/2026-06-10-core-notifications-*` and `*-push-*`.
|
||||
|
||||
---
|
||||
|
||||
## 1. Automated tests (fast, no live instance)
|
||||
|
||||
Run from the repo root. `export GOWORK=off` first (or run inside the worktree).
|
||||
|
||||
```bash
|
||||
# Go — materializer, matchers, worker state machine, request events, presence, handlers, tasks
|
||||
GOWORK=off go test ./internal/notifications/ ./internal/push/ ./internal/presence/ \
|
||||
./internal/requests/ ./internal/api/handlers/ ./internal/taskmanager/tasks/
|
||||
|
||||
# Web — bell, inbox, settings, admin page, usePushDevice, SW logic
|
||||
cd web && pnpm exec vitest run
|
||||
```
|
||||
|
||||
Expected: all green. The Go suites cover the outcome state machine (sent/skipped/failed-with-backoff/dead + dead-token prune), child-suppression SQL, the `notifications.send` plugin contract, request lifecycle publishing, and presence refcounting. The web suites cover the UI and the `usePushDevice` lifecycle/race-guard.
|
||||
|
||||
What this does NOT cover: real WebSocket delivery, real push banners, real migrations against production-shaped data. Use layers 2–3 for those.
|
||||
|
||||
---
|
||||
|
||||
## 2. Server pipeline (API — proves source → materializer → enqueuer → worker, no browser)
|
||||
|
||||
Needs an admin API key and the instance reachable (on silo-new the API is `http://localhost:8090`; create a temp key, delete it after — see the snippet at the bottom). Observe results in the `notifications` and `push_deliveries` tables.
|
||||
|
||||
For each notification **source**, trigger it and confirm a row lands (and, if push is configured + a device registered, a `push_deliveries` row):
|
||||
|
||||
- [ ] **Announcement** — `POST /api/v1/admin/announcements {title, body, audience:{all:true}}` → confirm a `notifications` row (category `announcement`) for each targeted user. ⚠️ `audience:{all:true}` fans out to **every** user's inbox — use `{user_ids:[<your id>]}` when testing on a live instance.
|
||||
- [ ] **Request lifecycle** — submit a media request, then `POST /api/v1/requests/{id}/approve` (admin) → confirm a `request.approved` notification for the requester. Decline/complete/fail similarly produce `request.declined` / `request.completed` / `request.failed`.
|
||||
- [ ] **System** — admin user-update that changes a password → confirm a `system.password_changed` notification for that user.
|
||||
- [ ] **Content** — add an item to a profile's watchlist/favorites, then add/scan a matching library item → confirm a `content.added` notification for that profile (respects the 48h recency gate + per-day burst dedup).
|
||||
- [ ] **Admin alert** — force a failed job/scan → confirm an `admin`-category notification fans out to admin users only (throttled 1/hour per source).
|
||||
- [ ] **Digest** — opt a user into `content_digest`, then run the `notifications_digest` task (or wait for 08:00) → one digest notification if there were additions.
|
||||
- [ ] **Push test** — `POST /api/v1/admin/push/test` → 202; a `system.test_push` notification + a `push_deliveries` row per the caller's registered push devices; the worker transitions it within ~30–45s (grace + tick). With a real subscription it sends; with a bogus one it fails-with-backoff (still proves the worker path).
|
||||
|
||||
Push config endpoints to verify:
|
||||
- [ ] `GET /api/v1/admin/push/status` → `{apns,fcm,webpush}` booleans (flip after saving creds).
|
||||
- [ ] `POST /api/v1/admin/push/generate-vapid-keys` → `{vapid_public, vapid_private}` (admin only).
|
||||
- [ ] `GET /api/v1/notifications/push/webpush-key` → the configured public key (empty when unconfigured).
|
||||
- [ ] `PUT /api/v1/notifications/push/device {transport:"webpush", token:<subscription JSON>}` with `X-Profile-Id` → 204 (400 without the profile header — the route is profile-gated).
|
||||
- [ ] `GET /api/v1/notifications/push/devices` → lists the user's push devices; `PUT .../devices/{id} {enabled}` toggles.
|
||||
|
||||
---
|
||||
|
||||
## 3. Real browser (the only way to validate the UI + actual push banner)
|
||||
|
||||
Open the web app over a secure context (`localhost` or HTTPS — service workers require it). On silo-new that's `http://localhost:8090` (tunnel if remote).
|
||||
|
||||
In-app notifications:
|
||||
- [ ] **Bell + badge** — top of the app; unread count badge.
|
||||
- [ ] **Dropdown** — latest notifications, "Mark all read", deep links.
|
||||
- [ ] **Inbox page** (`/notifications`) — category tabs, dismiss, mark-on-view, "Load more".
|
||||
- [ ] **Live toast** — with the app open, trigger a notification (announcement to yourself, or the push-test) → a toast appears in real time (via the WebSocket), unless that category is muted.
|
||||
- [ ] **Preferences** — Settings → Notifications → per-category mute toggles persist; "Daily digest" defaults off; the admin category shows only for admins.
|
||||
|
||||
Push (the headline manual test):
|
||||
- [ ] **Admin config** — Admin → Settings → Push Notifications → "Generate keys", save the VAPID public/private + a `mailto:` subject → the Web Push badge flips to "Ready".
|
||||
- [ ] **Enable on this device** — Settings → Notifications → "Push on this device" → grant the browser permission prompt → a device row appears in the list.
|
||||
- [ ] **Background banner** — **close the app tab**, then `POST /admin/push/test` (or trigger any notification) → within ~30–45s an OS notification banner appears.
|
||||
- [ ] **Click-through** — click the banner → the app opens/focuses and navigates to the notification's deep link.
|
||||
- [ ] **Presence gating** — with the app tab open and focused, the same notification should NOT push a redundant banner (you saw it in-app); after closing the tab for >30s it does.
|
||||
- [ ] **Per-device toggle / revoke** — disable the device in the list → no more banners; re-enable restores.
|
||||
|
||||
---
|
||||
|
||||
## 4. Not yet testable (tracked future work)
|
||||
|
||||
- **APNs (Apple) / FCM (Android)** — no mobile clients exist to register tokens, so these transports are only unit-tested (pure result-mappers). End-to-end needs silo-apple / silo-android (Phase 3 mobile).
|
||||
- **Plugin `notifications.send`** — the ebook/audiobook request plugins don't publish it yet. Until those one-line `PublishEvent("notifications.send", …)` calls are added, plugin-sourced notifications can't be exercised end to end (the contract itself is unit-tested).
|
||||
- **iOS Safari web push** — only works for an installed PWA, not a normal tab; the client degrades to an "unsupported" state otherwise.
|
||||
|
||||
---
|
||||
|
||||
## Appendix — temp admin key on silo-new (create, use, delete)
|
||||
|
||||
```bash
|
||||
# create
|
||||
KEY="sa_$(openssl rand -hex 32)"
|
||||
docker exec silo-server-postgres-1 psql -U silo -d silo -tAc \
|
||||
"INSERT INTO api_keys (user_id,label,api_key) VALUES (1,'manual-test','$KEY');"
|
||||
echo "$KEY" # use as: Authorization: Bearer $KEY against http://localhost:8090/api/v1/...
|
||||
|
||||
# observe
|
||||
docker exec silo-server-postgres-1 psql -U silo -d silo -tAc \
|
||||
"SELECT id,category,type,title FROM notifications ORDER BY id DESC LIMIT 10;"
|
||||
docker exec silo-server-postgres-1 psql -U silo -d silo -tAc \
|
||||
"SELECT status,transport,attempts,left(last_error,40) FROM push_deliveries ORDER BY id DESC LIMIT 10;"
|
||||
|
||||
# clean up (delete the key + any test artifacts you created)
|
||||
docker exec silo-server-postgres-1 psql -U silo -d silo -tAc \
|
||||
"DELETE FROM api_keys WHERE label='manual-test';"
|
||||
```
|
||||
|
||||
Always remove test notifications/devices you create on a real user's account; `push_deliveries` cascades when its `notifications` row is deleted.
|
||||
@@ -17,14 +17,14 @@
|
||||
| Path | Created/Modified | Purpose |
|
||||
|---|---|---|
|
||||
| `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` | Create | Findings doc produced by Task 1. Locks the remaining schema/code decisions. |
|
||||
| `migrations/139_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/139_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/140_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/140_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/141_media_libraries_kind.up.sql` | Create (conditional — see Task 4 step 1) | Add `kind` column to `media_libraries`. |
|
||||
| `migrations/141_media_libraries_kind.down.sql` | Create (same condition) | Drop the column. |
|
||||
| `migrations/142_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/142_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `migrations/147_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/147_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/157_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/157_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/159_media_folders_kind_noop.up.sql` | Create (conditional — see Task 4 step 1) | Document `media_folders.type='audiobooks'`; no column change needed. |
|
||||
| `migrations/159_media_folders_kind_noop.down.sql` | Create (same condition) | No-op rollback. |
|
||||
| `migrations/160_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/160_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `internal/audiobooks/doc.go` | Create | Package comment + intent. |
|
||||
| `internal/audiobooks/service.go` | Create | Empty `Service` struct, `New` constructor, `Enabled()` reader. |
|
||||
| `internal/audiobooks/service_test.go` | Create | One unit test proving the package compiles and `Enabled()` returns the setting value. |
|
||||
@@ -240,15 +240,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Migration 139 — `abs_sessions`
|
||||
## Task 2: Migration 147 — `abs_sessions`
|
||||
|
||||
Parallel of `jellycompat_sessions` for Audiobookshelf clients. Lets ABS
|
||||
mobile/desktop apps reconnect without re-authenticating against silo's
|
||||
main `auth_sessions`.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/139_abs_sessions.up.sql`
|
||||
- Create: `migrations/139_abs_sessions.down.sql`
|
||||
- Create: `migrations/147_abs_sessions.up.sql`
|
||||
- Create: `migrations/147_abs_sessions.down.sql`
|
||||
|
||||
> **Note:** If Task 1, Step 1.2 found the next number is not 139, rename
|
||||
> both filenames to match. All other tasks downstream of 139 shift by the
|
||||
@@ -266,7 +266,7 @@ main `auth_sessions`.
|
||||
|
||||
### Step 2.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/139_abs_sessions.up.sql` with:
|
||||
- [ ] **Create** `migrations/147_abs_sessions.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Audiobookshelf-compatible client sessions. Parallel to
|
||||
@@ -277,7 +277,7 @@ main `auth_sessions`.
|
||||
CREATE TABLE IF NOT EXISTS public.abs_sessions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
|
||||
token TEXT NOT NULL,
|
||||
token_hash TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
device_name TEXT,
|
||||
client_name TEXT,
|
||||
@@ -287,8 +287,8 @@ main `auth_sessions`.
|
||||
revoked_at TIMESTAMP WITH TIME ZONE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token
|
||||
ON public.abs_sessions (token);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token_hash
|
||||
ON public.abs_sessions (token_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_abs_sessions_user_device
|
||||
ON public.abs_sessions (user_id, device_id);
|
||||
@@ -299,12 +299,12 @@ main `auth_sessions`.
|
||||
|
||||
### Step 2.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/139_abs_sessions.down.sql` with:
|
||||
- [ ] **Create** `migrations/147_abs_sessions.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_last_seen;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_user_device;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token_hash;
|
||||
DROP TABLE IF EXISTS public.abs_sessions;
|
||||
```
|
||||
|
||||
@@ -328,27 +328,27 @@ main `auth_sessions`.
|
||||
- [ ] **Run** (each command should now succeed):
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 139;"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 147;"
|
||||
```
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=139`.
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=147`.
|
||||
|
||||
- [ ] **Run** a sample insert/delete to prove constraints work:
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
INSERT INTO abs_sessions (user_id, token, device_id, device_name, client_name, client_version)
|
||||
VALUES (1, 'test-token-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token = 'test-token-001';
|
||||
INSERT INTO abs_sessions (user_id, token_hash, device_id, device_name, client_name, client_version)
|
||||
VALUES ((SELECT id FROM users ORDER BY id LIMIT 1), 'test-token-hash-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token_hash = 'test-token-hash-001';
|
||||
"
|
||||
```
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (User id 1 must exist — silo's first user.)
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (At least one user must exist.)
|
||||
|
||||
### Step 2.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/139_abs_sessions.up.sql migrations/139_abs_sessions.down.sql
|
||||
git add migrations/147_abs_sessions.up.sql migrations/147_abs_sessions.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 139 add abs_sessions table
|
||||
feat(audiobooks): migration 147 add abs_sessions table
|
||||
|
||||
Parallel of jellycompat_sessions for Audiobookshelf-compatible clients.
|
||||
Lets ABS mobile/desktop apps maintain a device-bound session that
|
||||
@@ -361,15 +361,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Migration 140 — `podcast_feeds`
|
||||
## Task 3: Migration 157 — `podcast_feeds`
|
||||
|
||||
One row per subscribed podcast. The podcast itself lives in `media_items`
|
||||
(`type='podcast'`); this side table carries the RSS-specific metadata the
|
||||
feed refresher needs.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/140_podcast_feeds.up.sql`
|
||||
- Create: `migrations/140_podcast_feeds.down.sql`
|
||||
- Create: `migrations/157_podcast_feeds.up.sql`
|
||||
- Create: `migrations/157_podcast_feeds.down.sql`
|
||||
|
||||
### Step 3.1: Write the failing assertion
|
||||
|
||||
@@ -381,7 +381,7 @@ feed refresher needs.
|
||||
|
||||
### Step 3.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/140_podcast_feeds.up.sql` with:
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- RSS-feed metadata for subscribed podcasts. One row per podcast
|
||||
@@ -416,7 +416,7 @@ feed refresher needs.
|
||||
|
||||
### Step 3.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/140_podcast_feeds.down.sql` with:
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_podcast_feeds_due_for_refresh;
|
||||
@@ -438,17 +438,17 @@ feed refresher needs.
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 140;"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 157;"
|
||||
```
|
||||
Expected: column list printed; one row with `version=140`.
|
||||
Expected: column list printed; one row with `version=157`.
|
||||
|
||||
### Step 3.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/140_podcast_feeds.up.sql migrations/140_podcast_feeds.down.sql
|
||||
git add migrations/157_podcast_feeds.up.sql migrations/157_podcast_feeds.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 140 add podcast_feeds table
|
||||
feat(audiobooks): migration 157 add podcast_feeds table
|
||||
|
||||
Side table on media_items for RSS-subscribed podcasts. Holds feed URL,
|
||||
ETag/Last-Modified for conditional fetches, last-refresh timestamp, and
|
||||
@@ -462,7 +462,7 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Migration 141 — `media_libraries.kind` (conditional)
|
||||
## Task 4: Migration 159 — `media_libraries.kind` audiobook value (no-op)
|
||||
|
||||
Adds a discriminator column so the scanner can branch on
|
||||
`audiobooks`/`podcasts` libraries without filename heuristics.
|
||||
@@ -474,8 +474,8 @@ Adds a discriminator column so the scanner can branch on
|
||||
> Otherwise, proceed with the steps below.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/141_media_libraries_kind.up.sql`
|
||||
- Create: `migrations/141_media_libraries_kind.down.sql`
|
||||
- Create: `migrations/159_media_folders_kind_noop.up.sql`
|
||||
- Create: `migrations/159_media_folders_kind_noop.down.sql`
|
||||
|
||||
### Step 4.1: Write the failing assertion
|
||||
|
||||
@@ -487,7 +487,7 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
### Step 4.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/141_media_libraries_kind.up.sql` with:
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Per-library content discriminator. Lets the scanner choose the right
|
||||
@@ -509,7 +509,7 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
### Step 4.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/141_media_libraries_kind.down.sql` with:
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_media_libraries_kind;
|
||||
@@ -551,9 +551,9 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/141_media_libraries_kind.up.sql migrations/141_media_libraries_kind.down.sql
|
||||
git add migrations/159_media_folders_kind_noop.up.sql migrations/159_media_folders_kind_noop.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 141 add media_libraries.kind
|
||||
feat(audiobooks): migration 159 document audiobooks media folder type
|
||||
|
||||
Per-library discriminator that lets the scanner pick the right parser
|
||||
('movies', 'tv', 'audiobooks', 'podcasts') instead of inferring from
|
||||
@@ -567,15 +567,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Migration 142 — `audiobooks.enabled` feature flag
|
||||
## Task 5: Migration 160 — `audiobooks.enabled` feature flag
|
||||
|
||||
Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
"audiobooks compiled in but turned off" vs "fully live". Default is
|
||||
`false` so this sub-plan's landing is a strict no-op for users.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/142_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/142_audiobooks_feature_flag.down.sql`
|
||||
- Create: `migrations/160_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/160_audiobooks_feature_flag.down.sql`
|
||||
|
||||
### Step 5.1: Write the failing assertion
|
||||
|
||||
@@ -589,11 +589,11 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
### Step 5.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/142_audiobooks_feature_flag.up.sql` with:
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Master kill-switch for the absorbed audiobooks feature. Defaults to
|
||||
-- 'false' so landing migrations 139-142 is a no-op for users; operators
|
||||
-- 'false' so landing migrations 147/157/159/160 is a no-op for users; operators
|
||||
-- flip this to 'true' at cutover.
|
||||
|
||||
INSERT INTO server_settings (key, value) VALUES ('audiobooks.enabled', 'false')
|
||||
@@ -602,7 +602,7 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
### Step 5.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/142_audiobooks_feature_flag.down.sql` with:
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.down.sql` with:
|
||||
|
||||
```sql
|
||||
DELETE FROM server_settings WHERE key = 'audiobooks.enabled';
|
||||
@@ -625,9 +625,9 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/142_audiobooks_feature_flag.up.sql migrations/142_audiobooks_feature_flag.down.sql
|
||||
git add migrations/160_audiobooks_feature_flag.up.sql migrations/160_audiobooks_feature_flag.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 142 add audiobooks.enabled flag
|
||||
feat(audiobooks): migration 160 add audiobooks.enabled flag
|
||||
|
||||
Server-settings row that gates the absorbed audiobooks feature.
|
||||
Defaults to 'false' so sub-plan 1 lands as a strict no-op; subsequent
|
||||
@@ -646,7 +646,7 @@ EOF
|
||||
Creates an empty but compiling Go package, wires its constructor into
|
||||
`cmd/silo/main.go` so the package is referenced from the binary, and
|
||||
ships a unit test that proves the `Enabled()` reader returns the flag
|
||||
value seeded by migration 142.
|
||||
value seeded by migration 160.
|
||||
|
||||
This task proves the package compiles inside the rest of silo without
|
||||
changing any runtime behavior. No routes are mounted. No scheduled
|
||||
@@ -721,7 +721,7 @@ tasks are registered. No DB writes happen.
|
||||
|
||||
### Step 6.2: Run the test to verify it fails
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/...`
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
|
||||
Expected: build failure with messages like `undefined: New` and
|
||||
`package audiobooks not found`.
|
||||
@@ -777,9 +777,9 @@ tasks are registered. No DB writes happen.
|
||||
return &Service{settings: settings}
|
||||
}
|
||||
|
||||
// Enabled reports whether the audiobooks feature flag (set by migration
|
||||
// 142 and toggled by operators) is currently true. Any value other than
|
||||
// the literal string "true" reads as false; this matches how silo
|
||||
// Enabled reports whether the audiobooks feature flag (set by
|
||||
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
|
||||
// Any value other than the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
@@ -795,7 +795,7 @@ tasks are registered. No DB writes happen.
|
||||
|
||||
### Step 6.5: Run the test to verify it passes
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/...`
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
Expected: `ok github.com/Silo-Server/silo-server/internal/audiobooks ...`
|
||||
|
||||
### Step 6.6: Wire the constructor into `cmd/silo/main.go`
|
||||
@@ -831,13 +831,13 @@ are constructed and pass a settings store to.
|
||||
|
||||
### Step 6.7: Verify everything still compiles and tests pass
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go build ./...`
|
||||
- [ ] **Run:** `go build ./...`
|
||||
Expected: exit 0, no output.
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/... ./cmd/...`
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/... ./cmd/...`
|
||||
Expected: all passes.
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && make lint`
|
||||
- [ ] **Run:** `make lint`
|
||||
Expected: no new lint findings introduced by this task.
|
||||
|
||||
### Step 6.8: Smoke-test the running binary
|
||||
@@ -883,9 +883,9 @@ EOF
|
||||
| Spec section | Task(s) that cover it |
|
||||
|---|---|
|
||||
| Data model: reused tables | Task 1 (verification only) |
|
||||
| Data model: `abs_sessions` (mig 134→139) | Task 2 |
|
||||
| Data model: `podcast_feeds` (mig 135→140) | Task 3 |
|
||||
| Data model: `media_libraries.kind` (mig 136→141) | Task 4 |
|
||||
| Data model: `abs_sessions` (migration 147) | Task 2 |
|
||||
| Data model: `podcast_feeds` (migration 157) | Task 3 |
|
||||
| Data model: `media_folders.type` audiobook value (migration 159 no-op) | Task 4 |
|
||||
| Data model: data migration = none | n/a — confirmed in spec |
|
||||
| Feature flag (`audiobooks.enabled`) | Task 5 |
|
||||
| Architecture: `internal/audiobooks/` package | Task 6 |
|
||||
@@ -909,9 +909,8 @@ fires).
|
||||
test matches the production interface defined in `service.go`. Method
|
||||
name `GetString`, signature `(ctx context.Context, key string) (string, error)` — consistent across both files.
|
||||
|
||||
**Note on migration numbering:** Tasks 2–5 use numbers `139..142` based
|
||||
on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. Task 1 Step 1.2 re-verifies this; if
|
||||
the executing agent finds the highest existing number has moved, it
|
||||
must shift filenames in Tasks 2–5 by the same delta and update the
|
||||
matching task headings and commit-message references.
|
||||
**Note on migration numbering:** Tasks 2–5 originally used numbers `139..142`
|
||||
based on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make silo's existing scanner recognize audiobook + podcast libraries (`media_folders.type='audiobooks'` or `'podcasts'`), walk them to find audio files, and write `media_items.type='audiobook'` / `media_items.type='podcast'` rows with chapter metadata and author/narrator role links. No external metadata enrichment in this sub-plan — file tags only.
|
||||
**Goal:** Make silo's existing scanner recognize audiobook + podcast libraries (`media_folders.type='audiobooks'` or `'podcasts'`), walk them to find audio files, and write `media_items.type='audiobook'` / `media_items.type='podcast'` rows with chapter metadata and author/narrator role links. No external metadata enrichment runs in this sub-plan; the scanner extracts local tags and identity hints, and the audiobook metadata plugin/provider enriches canonical metadata later.
|
||||
|
||||
**Architecture:** New constants (`PersonKindAuthor=7`, `PersonKindNarrator=8`, `audioExtensions`). New helpers (`isAudiobookLibraryType`, `isPodcastLibraryType`, `SupportsAudioFile`). New parser file `internal/scanner/audiobook.go` for the folder-to-media-items mapping. The chapter extraction reuses the existing `probe.go` ffprobe helper. Author/narrator names are upserted into the existing `people` and `item_people` tables via the same code paths used for movie actors. No changes to the matching/enrichment metadata queues — those are deferred.
|
||||
**Architecture:** New constants (`PersonKindAuthor=7`, `PersonKindNarrator=8`, `audioExtensions`). New helpers (`isAudiobookLibraryType`, `isPodcastLibraryType`, `SupportsAudioFile`). New parser file `internal/scanner/audiobook.go` for the folder-to-media-items mapping. The chapter extraction reuses the existing `probe.go` ffprobe helper. Author/narrator names are upserted into the existing `people` and `item_people` tables via the same code paths used for movie actors. Matching/enrichment queue integration for the audiobook metadata plugin/provider is deferred.
|
||||
|
||||
**Tech Stack:** Go 1.26 + pgx; ffprobe via `os/exec` (already vendored as `internal/scanner/probe.go`); ID3/MP4 tag parsing — verify what silo already vendors before introducing a new library (acceptable: `github.com/dhowden/tag` if not already present, OR shell out to ffprobe which exposes tag dictionaries).
|
||||
|
||||
@@ -408,8 +408,8 @@ ffmpeg -y -f lavfi -i "anullsrc=r=22050:cl=mono" -t 10 -metadata title="Test Aud
|
||||
-metadata artist="Test Author" -metadata album="Test Series" \
|
||||
-metadata:s:a:0 language=eng \
|
||||
internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a
|
||||
# Add chapters via a temporary metadata file
|
||||
cat > /tmp/chapters.ffmeta <<'EOF'
|
||||
# Add chapters via a repo-local metadata fixture
|
||||
cat > internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta <<'EOF'
|
||||
;FFMETADATA1
|
||||
[CHAPTER]
|
||||
TIMEBASE=1/1000
|
||||
@@ -423,9 +423,9 @@ END=10000
|
||||
title=Outro
|
||||
EOF
|
||||
ffmpeg -y -i internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a \
|
||||
-i /tmp/chapters.ffmeta -map_metadata 1 -codec copy \
|
||||
-i internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta -map_metadata 1 -codec copy \
|
||||
internal/scanner/testdata/audiobook_fixtures/single_book/book.m4b
|
||||
rm internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a /tmp/chapters.ffmeta
|
||||
rm internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta
|
||||
```
|
||||
|
||||
Add a test that calls the chapter-extraction function on `book.m4b` and asserts:
|
||||
@@ -1008,7 +1008,6 @@ Verify the full sub-plan landed cleanly.
|
||||
### Step 10.1: Full tests + build
|
||||
|
||||
```bash
|
||||
cd /opt/silo-server
|
||||
go build ./...
|
||||
go test ./internal/scanner/... ./internal/models/... ./internal/audiobooks/...
|
||||
go vet ./...
|
||||
@@ -1019,7 +1018,7 @@ All green.
|
||||
### Step 10.2: Rebuild + recreate silo container
|
||||
|
||||
```bash
|
||||
sudo docker build -t silo:latest /opt/silo-server
|
||||
sudo docker build -t silo:latest .
|
||||
sudo docker compose -p silo-prod up -d --force-recreate silo
|
||||
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
|
||||
```
|
||||
@@ -1065,7 +1064,7 @@ Otherwise: no commit needed.
|
||||
| Multi-file audiobooks | T7 |
|
||||
| Filesystem podcasts | T9 |
|
||||
| RSS podcasts | deferred to sub-plan 5 |
|
||||
| External enrichment (OpenLibrary/Google Books) | dropped per spec |
|
||||
| External enrichment through audiobook metadata plugin/provider | deferred per spec |
|
||||
|
||||
**Placeholder scan:** Task 8 has language like "grep for upsertMediaItem or similar in scanner.go to find it" — this is intentional. The scanner's write helpers are too deeply embedded for me to enumerate without reading more code; the implementer subagent will discover them. If a subagent reports BLOCKED on Task 8 because the write helpers aren't shaped how this plan assumes, the controller dispatches a discovery sub-task.
|
||||
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
**Goal:** Port the plugin's `podcastfeed` package as a first-party scheduled task in silo, so RSS-subscribed podcasts get their episode lists kept up to date.
|
||||
|
||||
**Source plugin:** `/opt/silo_plugins/silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
**Source plugin:** `../silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1: Port the refresher package
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `/opt/silo-server/internal/audiobooks/podcastfeed/`
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `internal/audiobooks/podcastfeed/`
|
||||
- Update imports (drop plugin SDK; use silo's `pgxpool` + `log/slog`)
|
||||
- Rewrite SQL queries against silo's `podcast_feeds` table (created in sub-plan 1 migration 140), `media_items` (type='podcast'), and `episodes`
|
||||
- Drop the plugin's `presentation_libraries` reference; subscriptions are tracked by `podcast_feeds.media_item_id`
|
||||
|
||||
@@ -0,0 +1,724 @@
|
||||
# Resilient Library Deletion Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Rewrite `delete_library` so it deletes a library in small, autocommitted, deadlock-retrying batches instead of one multi-minute transaction that deadlocks and rolls back on large libraries.
|
||||
|
||||
**Architecture:** `FolderRepository.DeleteWithStats` becomes a phased, batched operation (orphan items → folder media_files → remaining shared junctions → folder row). Each batch is one autocommit statement wrapped in a `retryOnDeadlock` helper, so locks are held briefly and the job is resumable. Two small helpers (`retryOnDeadlock`, `deleteInBatches`) are unit-tested in pure Go; the full path is verified against the live dev database.
|
||||
|
||||
**Tech Stack:** Go, pgx v5 (`pgxpool`, `pgconn`), PostgreSQL. Spec: `docs/superpowers/specs/2026-05-28-library-delete-resilience-design.md`.
|
||||
|
||||
All commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- **Modify** `internal/catalog/folder_repo.go`
|
||||
- Add constants `orphanDeleteBatch`, `folderChildDeleteBatch`.
|
||||
- Add package vars `deadlockMaxAttempts`, `deadlockBaseBackoff` and func `retryOnDeadlock`.
|
||||
- Add func `deleteInBatches`.
|
||||
- Add interface `rowQuerier`; change `collectImageDirs` and `filterUnreferencedImageDirs` to take `rowQuerier` instead of `pgx.Tx`; extract `collectRawImageDirs`.
|
||||
- Rewrite `DeleteWithStats`; add helpers `collectOrphanBatch`, `dirSetToSlice`.
|
||||
- Leave `collectOrphanIDs` and `Delete` unchanged.
|
||||
- **Create** `internal/catalog/folder_delete_test.go` — pure-Go unit tests for `retryOnDeadlock` and `deleteInBatches`.
|
||||
- **Do not modify** `internal/catalog/library_repo.go` — its `collectImageDirs`/`collectOrphanIDs` calls keep compiling because `pgx.Tx` satisfies `rowQuerier`. (Verified by build.)
|
||||
- **Unchanged behavior** in `internal/adminjob/library_delete.go` (executor), `internal/adminjob/runner.go` (heartbeat).
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Deadlock-retry helper
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go`
|
||||
- Test: `internal/catalog/folder_delete_test.go` (create)
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Create `internal/catalog/folder_delete_test.go`:
|
||||
|
||||
```go
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// withFastDeadlockRetry shrinks retry timing/attempts for tests and restores
|
||||
// the originals on cleanup. Tests using it must not call t.Parallel().
|
||||
func withFastDeadlockRetry(t *testing.T, maxAttempts int) {
|
||||
t.Helper()
|
||||
oldMax, oldBackoff := deadlockMaxAttempts, deadlockBaseBackoff
|
||||
deadlockMaxAttempts = maxAttempts
|
||||
deadlockBaseBackoff = time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
deadlockMaxAttempts = oldMax
|
||||
deadlockBaseBackoff = oldBackoff
|
||||
})
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockRetriesThenSucceeds(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
calls := 0
|
||||
err := retryOnDeadlock(context.Background(), func() error {
|
||||
calls++
|
||||
if calls < 3 {
|
||||
return &pgconn.PgError{Code: "40P01"}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected success, got %v", err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("expected 3 calls, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockReturnsNonRetryableImmediately(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
sentinel := errors.New("boom")
|
||||
calls := 0
|
||||
err := retryOnDeadlock(context.Background(), func() error {
|
||||
calls++
|
||||
return sentinel
|
||||
})
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("expected sentinel, got %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("expected 1 call, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockGivesUpAfterMaxAttempts(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 4)
|
||||
calls := 0
|
||||
err := retryOnDeadlock(context.Background(), func() error {
|
||||
calls++
|
||||
return &pgconn.PgError{Code: "40P01"}
|
||||
})
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) || pgErr.Code != "40P01" {
|
||||
t.Fatalf("expected 40P01 pg error, got %v", err)
|
||||
}
|
||||
if calls != 4 {
|
||||
t.Fatalf("expected 4 calls, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryOnDeadlockStopsOnCanceledContext(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
calls := 0
|
||||
err := retryOnDeadlock(ctx, func() error {
|
||||
calls++
|
||||
return &pgconn.PgError{Code: "40P01"}
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled, got %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("expected 1 call before cancel, got %d", calls)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestRetryOnDeadlock -v`
|
||||
Expected: compile failure / FAIL — `undefined: retryOnDeadlock`, `deadlockMaxAttempts`, `deadlockBaseBackoff`.
|
||||
|
||||
- [ ] **Step 3: Implement the helper**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add near the top of the file (after the `import` block, before the sentinel errors var is fine):
|
||||
|
||||
```go
|
||||
// Retry parameters for transient serialization/deadlock failures. They are
|
||||
// package vars (not consts) only so tests can shrink them; production code
|
||||
// never mutates them.
|
||||
var (
|
||||
deadlockMaxAttempts = 5
|
||||
deadlockBaseBackoff = 50 * time.Millisecond
|
||||
)
|
||||
|
||||
// retryOnDeadlock runs op, retrying when Postgres reports a deadlock (40P01) or
|
||||
// serialization failure (40001), with exponential backoff. It returns
|
||||
// immediately for any other error, and honors context cancellation between
|
||||
// attempts.
|
||||
func retryOnDeadlock(ctx context.Context, op func() error) error {
|
||||
backoff := deadlockBaseBackoff
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := op()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if attempt < deadlockMaxAttempts && errors.As(err, &pgErr) &&
|
||||
(pgErr.Code == "40P01" || pgErr.Code == "40001") {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
backoff *= 2
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
(`context`, `errors`, `time`, and `github.com/jackc/pgx/v5/pgconn` are already imported in this file.)
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestRetryOnDeadlock -v`
|
||||
Expected: PASS (4 tests).
|
||||
|
||||
- [ ] **Step 5: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git add internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git commit -m "feat(catalog): add deadlock-retry helper for batched deletes"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Batched-delete loop helper
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go`
|
||||
- Test: `internal/catalog/folder_delete_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
Append to `internal/catalog/folder_delete_test.go`:
|
||||
|
||||
```go
|
||||
func TestDeleteInBatchesLoopsUntilUnderBatchSize(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
counts := []int64{5, 5, 2}
|
||||
idx := 0
|
||||
total, err := deleteInBatches(context.Background(), 5, func(context.Context) (int64, error) {
|
||||
n := counts[idx]
|
||||
idx++
|
||||
return n, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if total != 12 {
|
||||
t.Fatalf("expected total 12, got %d", total)
|
||||
}
|
||||
if idx != 3 {
|
||||
t.Fatalf("expected 3 batches, got %d", idx)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteInBatchesStopsImmediatelyWhenFirstBatchUnderSize(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
calls := 0
|
||||
total, err := deleteInBatches(context.Background(), 5, func(context.Context) (int64, error) {
|
||||
calls++
|
||||
return 0, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if total != 0 || calls != 1 {
|
||||
t.Fatalf("expected total 0 and 1 call, got total=%d calls=%d", total, calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteInBatchesReturnsError(t *testing.T) {
|
||||
withFastDeadlockRetry(t, 5)
|
||||
sentinel := errors.New("delete failed")
|
||||
_, err := deleteInBatches(context.Background(), 5, func(context.Context) (int64, error) {
|
||||
return 0, sentinel
|
||||
})
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("expected sentinel, got %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests to verify they fail**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestDeleteInBatches -v`
|
||||
Expected: FAIL — `undefined: deleteInBatches`.
|
||||
|
||||
- [ ] **Step 3: Implement the helper**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add after `retryOnDeadlock`:
|
||||
|
||||
```go
|
||||
// deleteInBatches repeatedly runs deleteBatch (each a single autocommit
|
||||
// statement) until a batch removes fewer than batchSize rows. Each batch is
|
||||
// retried on deadlock. It returns the total number of rows deleted.
|
||||
func deleteInBatches(
|
||||
ctx context.Context,
|
||||
batchSize int,
|
||||
deleteBatch func(ctx context.Context) (int64, error),
|
||||
) (int64, error) {
|
||||
var total int64
|
||||
for {
|
||||
var affected int64
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
n, e := deleteBatch(ctx)
|
||||
affected = n
|
||||
return e
|
||||
}); err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += affected
|
||||
if affected < int64(batchSize) {
|
||||
return total, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `go test ./internal/catalog/ -run TestDeleteInBatches -v`
|
||||
Expected: PASS (3 tests).
|
||||
|
||||
- [ ] **Step 5: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git add internal/catalog/folder_repo.go internal/catalog/folder_delete_test.go
|
||||
git commit -m "feat(catalog): add deleteInBatches loop helper"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Generalize image-dir helpers to a querier interface
|
||||
|
||||
This lets the image-dir helpers run on either a `*pgxpool.Pool` (new batched path) or a `pgx.Tx` (existing `library_repo.go` path), and splits raw collection from filtering so the batched path can collect cheaply per batch and filter once.
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go:547-635` (the `collectImageDirs` / `filterUnreferencedImageDirs` block)
|
||||
|
||||
- [ ] **Step 1: Add the `rowQuerier` interface**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add (near the other type declarations, e.g. just above `collectOrphanIDs`):
|
||||
|
||||
```go
|
||||
// rowQuerier is satisfied by both *pgxpool.Pool and pgx.Tx, letting read
|
||||
// helpers run inside or outside an explicit transaction.
|
||||
type rowQuerier interface {
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Split `collectImageDirs` into raw collection + filtering**
|
||||
|
||||
Replace the existing `collectImageDirs` function (currently `internal/catalog/folder_repo.go:549-584`) with these two functions:
|
||||
|
||||
```go
|
||||
// collectImageDirs returns S3 directory prefixes for images belonging to the
|
||||
// given content IDs that are not still referenced by other surviving content.
|
||||
func collectImageDirs(ctx context.Context, q rowQuerier, contentIDs []string) ([]string, error) {
|
||||
dirs, err := collectRawImageDirs(ctx, q, contentIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return filterUnreferencedImageDirs(ctx, q, dirs, contentIDs)
|
||||
}
|
||||
|
||||
// collectRawImageDirs returns the deduped S3 directory prefixes referenced by
|
||||
// the given content IDs (items, their seasons, and their episodes), without
|
||||
// filtering out dirs still used by other content.
|
||||
func collectRawImageDirs(ctx context.Context, q rowQuerier, contentIDs []string) ([]string, error) {
|
||||
imgRows, err := q.Query(ctx, `
|
||||
SELECT poster_path, backdrop_path, logo_path FROM media_items WHERE content_id = ANY($1)
|
||||
UNION ALL
|
||||
SELECT poster_path, '', '' FROM seasons WHERE series_id = ANY($1)
|
||||
UNION ALL
|
||||
SELECT still_path, '', '' FROM episodes WHERE series_id = ANY($1)
|
||||
`, contentIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("collecting image paths: %w", err)
|
||||
}
|
||||
defer imgRows.Close()
|
||||
dirSet := make(map[string]struct{})
|
||||
for imgRows.Next() {
|
||||
var p1, p2, p3 string
|
||||
if err := imgRows.Scan(&p1, &p2, &p3); err != nil {
|
||||
return nil, fmt.Errorf("scanning image path: %w", err)
|
||||
}
|
||||
for _, p := range []string{p1, p2, p3} {
|
||||
if p != "" && !strings.Contains(p, "://") {
|
||||
if dir := pathDir(p); dir != "" {
|
||||
dirSet[dir] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := imgRows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterating image paths: %w", err)
|
||||
}
|
||||
dirs := make([]string, 0, len(dirSet))
|
||||
for dir := range dirSet {
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
return dirs, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Change `filterUnreferencedImageDirs` to take `rowQuerier`**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, change the signature (currently `internal/catalog/folder_repo.go:586`) from:
|
||||
|
||||
```go
|
||||
func filterUnreferencedImageDirs(ctx context.Context, tx pgx.Tx, dirs, deletingContentIDs []string) ([]string, error) {
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```go
|
||||
func filterUnreferencedImageDirs(ctx context.Context, q rowQuerier, dirs, deletingContentIDs []string) ([]string, error) {
|
||||
```
|
||||
|
||||
and change the single `tx.Query(ctx, ...)` call inside it to `q.Query(ctx, ...)`. Leave the SQL and the rest of the body unchanged.
|
||||
|
||||
- [ ] **Step 4: Verify the build (proves `library_repo.go` still compiles with `pgx.Tx`)**
|
||||
|
||||
Run: `go build ./... && go vet ./internal/catalog/`
|
||||
Expected: no errors. (`library_repo.go` passes a `pgx.Tx` to `collectImageDirs`, which now accepts `rowQuerier`; `pgx.Tx` satisfies it.)
|
||||
|
||||
- [ ] **Step 5: Run the catalog tests**
|
||||
|
||||
Run: `go test ./internal/catalog/...`
|
||||
Expected: PASS (existing tests plus Tasks 1–2 helpers).
|
||||
|
||||
- [ ] **Step 6: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go
|
||||
git add internal/catalog/folder_repo.go
|
||||
git commit -m "refactor(catalog): make image-dir helpers querier-agnostic"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Rewrite `DeleteWithStats` as a batched, phased operation
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/catalog/folder_repo.go:415-511` (the `DeleteWithStats` body) and add two helpers + two constants.
|
||||
|
||||
- [ ] **Step 1: Add the batch-size constants**
|
||||
|
||||
In `internal/catalog/folder_repo.go`, add near the retry vars from Task 1:
|
||||
|
||||
```go
|
||||
const (
|
||||
// orphanDeleteBatch is small because each media_items row cascades across
|
||||
// ~15 child tables.
|
||||
orphanDeleteBatch = 1000
|
||||
// folderChildDeleteBatch covers the lighter folder-scoped media_files and
|
||||
// junction deletes.
|
||||
folderChildDeleteBatch = 5000
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Replace the `DeleteWithStats` body**
|
||||
|
||||
Replace the entire existing `DeleteWithStats` function (`internal/catalog/folder_repo.go:415-511`) with:
|
||||
|
||||
```go
|
||||
func (r *FolderRepository) DeleteWithStats(
|
||||
ctx context.Context,
|
||||
id int,
|
||||
progress func(current, total int, message string),
|
||||
) (*DeleteFolderStats, error) {
|
||||
stats := &DeleteFolderStats{}
|
||||
|
||||
// Phase 0: preflight reads (no long-lived transaction).
|
||||
if err := r.pool.QueryRow(ctx, `SELECT name FROM media_folders WHERE id = $1`, id).Scan(&stats.LibraryName); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrFolderNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("loading folder before delete: %w", err)
|
||||
}
|
||||
if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM media_files WHERE media_folder_id = $1`, id).Scan(&stats.MediaFiles); err != nil {
|
||||
return nil, fmt.Errorf("counting media files: %w", err)
|
||||
}
|
||||
if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = $1`, id).Scan(&stats.MediaItemLinks); err != nil {
|
||||
return nil, fmt.Errorf("counting media item links: %w", err)
|
||||
}
|
||||
var orphanTotal int
|
||||
if err := r.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.media_folder_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries other
|
||||
WHERE other.content_id = mil.content_id
|
||||
AND other.media_folder_id <> $1
|
||||
)`, id).Scan(&orphanTotal); err != nil {
|
||||
return nil, fmt.Errorf("counting orphaned items: %w", err)
|
||||
}
|
||||
|
||||
// Phase 1: delete orphaned media_items in detect-then-delete batches.
|
||||
// Cascade removes their junctions, provider IDs, episodes, seasons, etc.
|
||||
if progress != nil {
|
||||
progress(0, orphanTotal, "Deleting orphaned items")
|
||||
}
|
||||
rawDirs := make(map[string]struct{})
|
||||
for {
|
||||
ids, err := r.collectOrphanBatch(ctx, id, orphanDeleteBatch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
dirs, err := collectRawImageDirs(ctx, r.pool, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, d := range dirs {
|
||||
rawDirs[d] = struct{}{}
|
||||
}
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
_, e := r.pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, ids)
|
||||
return e
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting orphaned items: %w", err)
|
||||
}
|
||||
stats.OrphanedItems += len(ids)
|
||||
if progress != nil {
|
||||
progress(stats.OrphanedItems, orphanTotal, "Deleting orphaned items")
|
||||
}
|
||||
}
|
||||
|
||||
// Filter accumulated image dirs once, now that orphans are gone, against
|
||||
// any surviving content. Empty deleting-set means "exclude nothing".
|
||||
if len(rawDirs) > 0 {
|
||||
filtered, err := filterUnreferencedImageDirs(ctx, r.pool, dirSetToSlice(rawDirs), []string{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats.OrphanedImageDirs = filtered
|
||||
}
|
||||
|
||||
// Phase 2: delete this folder's media_files (folder-tied, not item-tied).
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Deleting media files")
|
||||
}
|
||||
if _, err := deleteInBatches(ctx, folderChildDeleteBatch, func(ctx context.Context) (int64, error) {
|
||||
tag, e := r.pool.Exec(ctx, `
|
||||
DELETE FROM media_files
|
||||
WHERE id IN (
|
||||
SELECT id FROM media_files WHERE media_folder_id = $1 LIMIT $2
|
||||
)`, id, folderChildDeleteBatch)
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting media files: %w", err)
|
||||
}
|
||||
|
||||
// Phase 3: delete remaining folder memberships (shared items kept; only the
|
||||
// membership in this folder is removed).
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Removing library memberships")
|
||||
}
|
||||
if _, err := deleteInBatches(ctx, folderChildDeleteBatch, func(ctx context.Context) (int64, error) {
|
||||
tag, e := r.pool.Exec(ctx, `
|
||||
DELETE FROM media_item_libraries
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM media_item_libraries WHERE media_folder_id = $1 LIMIT $2
|
||||
)`, id, folderChildDeleteBatch)
|
||||
if e != nil {
|
||||
return 0, e
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting media item links: %w", err)
|
||||
}
|
||||
|
||||
// Phase 4: delete the now-lightweight folder row. Tolerate 0 rows so a
|
||||
// resumed run that already removed it still succeeds.
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
_, e := r.pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, id)
|
||||
return e
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("deleting folder: %w", err)
|
||||
}
|
||||
|
||||
if progress != nil {
|
||||
progress(orphanTotal, orphanTotal, "Library deletion completed")
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// collectOrphanBatch returns up to limit content IDs whose only library
|
||||
// membership is the given folder.
|
||||
func (r *FolderRepository) collectOrphanBatch(ctx context.Context, folderID, limit int) ([]string, error) {
|
||||
rows, err := r.pool.Query(ctx, `
|
||||
SELECT mil.content_id
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.media_folder_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries other
|
||||
WHERE other.content_id = mil.content_id
|
||||
AND other.media_folder_id <> $1
|
||||
)
|
||||
LIMIT $2`, folderID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("finding orphaned items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []string
|
||||
for rows.Next() {
|
||||
var contentID string
|
||||
if err := rows.Scan(&contentID); err != nil {
|
||||
return nil, fmt.Errorf("scanning orphan content_id: %w", err)
|
||||
}
|
||||
ids = append(ids, contentID)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
// dirSetToSlice returns the keys of set as a slice (nil if empty).
|
||||
func dirSetToSlice(set map[string]struct{}) []string {
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
dirs := make([]string, 0, len(set))
|
||||
for d := range set {
|
||||
dirs = append(dirs, d)
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
```
|
||||
|
||||
Note: this removes the old single-transaction body (`tx, err := r.pool.Begin(...)`, the inline orphan-collection loop, the `collectImageDirs` call, the CASCADE folder drop, and `tx.Commit`). `collectOrphanIDs` and `Delete` are untouched. `collectImageDirs` remains (still used by `library_repo.go`).
|
||||
|
||||
- [ ] **Step 3: Verify the build and vet**
|
||||
|
||||
Run: `go build ./... && go vet ./internal/catalog/`
|
||||
Expected: no errors. If `go vet` flags an unused function, confirm it is genuinely unused before removing — `collectImageDirs`, `collectOrphanIDs`, and `pathDir` must all remain (still referenced).
|
||||
|
||||
- [ ] **Step 4: Run the full catalog + adminjob test suites**
|
||||
|
||||
Run: `go test ./internal/catalog/... ./internal/adminjob/...`
|
||||
Expected: PASS. (No DB-backed test exercises `DeleteWithStats`; the helper unit tests and existing tests must stay green.)
|
||||
|
||||
- [ ] **Step 5: Lint**
|
||||
|
||||
Run: `golangci-lint run ./internal/catalog/...`
|
||||
Expected: no findings. (If `golangci-lint` is unavailable, run `gofmt -l internal/catalog/` and ensure it prints nothing.)
|
||||
|
||||
- [ ] **Step 6: Format and commit**
|
||||
|
||||
```bash
|
||||
gofmt -w internal/catalog/folder_repo.go
|
||||
git add internal/catalog/folder_repo.go
|
||||
git commit -m "fix(catalog): delete libraries in deadlock-retrying batches
|
||||
|
||||
Replaces the single multi-minute delete transaction with phased, batched
|
||||
autocommit deletes (orphan items, media files, memberships, folder row),
|
||||
each retried on deadlock. Holds only short locks, survives concurrent
|
||||
writers, and is resumable on failure."
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Verify on the dev server
|
||||
|
||||
This is the integration test (no catalog DB harness exists). Follow the deployment-debugging runbook. The Audiobooks library (folder id 8) is currently stuck with ~249K items / ~339K files.
|
||||
|
||||
- [ ] **Step 1: Deploy the branch to dev**
|
||||
|
||||
Run: `make dev-deploy`
|
||||
Expected: builds `silo:dev-local`, recreates the `silo` service. Then confirm health:
|
||||
`ssh "$DEV_HOST" 'curl -s http://localhost:8090/api/v1/ready'`
|
||||
|
||||
- [ ] **Step 2: Capture the starting counts**
|
||||
|
||||
Run (psql via the dev compose `postgres` service, user/db `continuum`):
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM media_files WHERE media_folder_id = 8) AS files,
|
||||
(SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = 8) AS links,
|
||||
(SELECT EXISTS (SELECT 1 FROM media_folders WHERE id = 8)) AS folder_exists;
|
||||
```
|
||||
|
||||
Expected: non-zero files/links, `folder_exists = t`.
|
||||
|
||||
- [ ] **Step 3: Trigger the deletion**
|
||||
|
||||
Re-trigger the library deletion through the normal path (admin UI "delete library" for Audiobooks, or the admin delete endpoint). Confirm a new job is queued:
|
||||
|
||||
```sql
|
||||
SELECT id, job_type, status, requested_at
|
||||
FROM admin_jobs WHERE job_type = 'delete_library'
|
||||
ORDER BY requested_at DESC LIMIT 3;
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Monitor to completion**
|
||||
|
||||
Watch the job and the draining counts (re-run periodically):
|
||||
|
||||
```sql
|
||||
SELECT id, status, progress_current, progress_total, error_message, heartbeat_at
|
||||
FROM admin_jobs WHERE job_type = 'delete_library'
|
||||
ORDER BY requested_at DESC LIMIT 1;
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = 8;
|
||||
```
|
||||
|
||||
Also confirm deadlock retries (if any) are recovered rather than fatal:
|
||||
`ssh "$DEV_HOST" "$DEV_COMPOSE_CMD logs postgres 2>&1 | grep -i deadlock | tail -20"`
|
||||
Expected: the job reaches `status = completed`; counts trend to 0; any deadlocks are transient (the job does not fail).
|
||||
|
||||
- [ ] **Step 5: Confirm full removal**
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM media_files WHERE media_folder_id = 8) AS files,
|
||||
(SELECT COUNT(*) FROM media_item_libraries WHERE media_folder_id = 8) AS links,
|
||||
(SELECT EXISTS (SELECT 1 FROM media_folders WHERE id = 8)) AS folder_exists;
|
||||
```
|
||||
|
||||
Expected: `files = 0`, `links = 0`, `folder_exists = f`.
|
||||
|
||||
- [ ] **Step 6: Report**
|
||||
|
||||
Summarize: starting counts, time to completion, number of batches/retries observed, and final state. No commit (verification only).
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:**
|
||||
- Phased batched deletion (Phases 0–4) → Task 4. ✔
|
||||
- `retryOnDeadlock` (40P01/40001, 5 attempts, backoff, ctx-aware) → Task 1. ✔
|
||||
- Batch sizes `orphanDeleteBatch=1000`, `folderChildDeleteBatch=5000` → Task 4 Step 1. ✔
|
||||
- Autocommit per batch / no long transaction → Task 4 Step 2 (uses `r.pool.Exec`, no `Begin`). ✔
|
||||
- Stats + progress + heartbeat reliance → Task 4 Step 2; runner heartbeat unchanged. ✔
|
||||
- Resumability (folder row last, tolerate 0 rows) → Task 4 Phase 4. ✔
|
||||
- Unit tests for retry + batch loop → Tasks 1–2. ✔
|
||||
- `collectImageDirs` shared with `library_repo.go` preserved → Task 3 (interface widening), Task 4 Step 3 build check. ✔
|
||||
- Dev integration verification → Task 5. ✔
|
||||
- Out of scope (writer coordination, runner/executor changes, schema changes) → not present in any task. ✔
|
||||
|
||||
**Placeholder scan:** No TBD/TODO; every code step has complete code; every command has expected output. ✔
|
||||
|
||||
**Type consistency:** `retryOnDeadlock(ctx, func() error) error`, `deleteInBatches(ctx, int, func(ctx) (int64,error)) (int64,error)`, `rowQuerier{ Query(ctx,string,...any)(pgx.Rows,error) }`, `collectRawImageDirs`/`collectImageDirs`/`filterUnreferencedImageDirs(ctx, rowQuerier, ...)`, `collectOrphanBatch(ctx,int,int)([]string,error)`, `dirSetToSlice(map[string]struct{})[]string`, consts `orphanDeleteBatch`/`folderChildDeleteBatch`, vars `deadlockMaxAttempts`/`deadlockBaseBackoff` — names used consistently across tasks. ✔
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,779 @@
|
||||
# Autoscan Host Backend Implementation Plan (Part 1 of 2)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build the silo-server backend for the standalone Autoscan category: poll `scan_source.v1` plugins on a timer, resolve their Silo-native paths to library folders, and enqueue rescans — with connections decoupled from Requests.
|
||||
|
||||
**Architecture:** Mirror the existing `scheduled_task.v1` host dispatch (`pluginhost.Client` capability wrapper + `plugins.Service` resolver + a `taskmanager` task). Generalize the closed PR #43 autoscan engine (resolve → suppress → enqueue) so its input is "Silo-native paths from a scan-source plugin" instead of "arr history." Replace the old `request_integrations`-coupled schema with `autoscan_connections` + `autoscan_sources`.
|
||||
|
||||
**Tech Stack:** Go, pgx/v5, the `internal/plugins`/`internal/pluginhost` runtime, `internal/scantrigger`/`internal/scanqueue`, `internal/taskmanager`, numbered SQL migrations, `github.com/Silo-Server/silo-plugin-sdk` (the new `scan_source.v1` capability).
|
||||
|
||||
Commands assume the silo-server repository root is the cwd. This is **Part 1 (backend)**; the Autoscan admin UI is Part 2.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **SDK dependency.** This plan needs the SDK's `scan_source.v1` symbols (`runtime.Client.ScanSource()`, `pluginv1.ScanSourceClient`, `pluginv1.PollChangesRequest/Response`, `capability.ScanSource`). They are in `silo-plugin-sdk` PR #2 but **not yet released**. Until `v0.5.0` is tagged, add a local replace directive so the host builds against the working checkout:
|
||||
|
||||
```bash
|
||||
# from the repository root (cwd)
|
||||
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk
|
||||
go mod tidy
|
||||
```
|
||||
|
||||
This points the replace at the `silo-plugin-sdk` checkout (a sibling of this repo).
|
||||
|
||||
The final task removes the replace and bumps to the tagged version.
|
||||
|
||||
2. **Closed PR #43 is on `main`.** The in-process arr autoscan (`internal/autoscan/*`, `internal/api/handlers/autoscan.go`, `internal/taskmanager/tasks/autoscan_poll.go`, router routes, migration `171_autoscan`, the `AdminRequests.tsx` tab) is the salvage source. This plan **generalizes the engine and replaces the schema/wiring**; it does not preserve the arr-specific history client (that moves to the arr plugin in a separate plan).
|
||||
|
||||
3. **Tests that touch `internal/api/handlers` or `internal/api` link libvips via CGO.** Run the suite in the libvips-equipped container (see project memory `silo-test-libvips-gotcha`), not the bare host, or those packages report `[build failed]` and skip silently.
|
||||
|
||||
---
|
||||
|
||||
## Pattern references (read before starting)
|
||||
|
||||
- Host capability client wrapper: `internal/pluginhost/client.go` — `ScheduledTaskClient` type (l.42), `Client.ScheduledTask` accessor (l.111), `requireCapability` (l.157), `Run` (l.226).
|
||||
- Host resolver: `internal/plugins/service.go:413` `ScheduledTaskClient(ctx, installationID, capabilityID)` → `ensureClient` → `client.ScheduledTask(capabilityID)`.
|
||||
- Timer task over plugin capabilities: `internal/plugins/task_registry.go` (iterate enabled installations, filter capability type, build a `taskmanager.Task`).
|
||||
- Salvageable engine (generalize, don't rewrite): `internal/autoscan/service.go` (resolve/suppress/enqueue loop), `dedupe.go` (`uniqueParentDirs`), `suppress.go` (`Suppressor`). Discard `history.go`, `rewrite.go`, `suggest*.go` (arr-specific → arr plugin).
|
||||
- Migration mechanics: paired `migrations/NNN_*.up.sql`/`.down.sql`, embedded via `migrations/embed.go`, auto-applied on boot.
|
||||
- Credential decryption for the reuse-from-Requests case: `internal/requests/repository.go` (`integrationColumns` has `base_url, api_key_ref`); Fernet resolution via `catalog.ServerSettingsRepo.Get` (the same path PR #43's `SecretResolver` used).
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `internal/pluginhost/client.go` | `ScanSourceClient` wrapper + `Client.ScanSource()` accessor | Modify |
|
||||
| `internal/pluginhost/client_test.go` | `requireCapability` gate for `scan_source.v1` | Modify |
|
||||
| `internal/plugins/service.go` | `ScanSourceClient(ctx, installationID, capabilityID)` resolver | Modify |
|
||||
| `migrations/172_autoscan_v2.up.sql` / `.down.sql` | `autoscan_connections` + `autoscan_sources`; drop the `171` tables | Create |
|
||||
| `internal/autoscan/types.go` | `Source`, `Connection`, `Settings`, `PollChanges` provider seam | Rewrite |
|
||||
| `internal/autoscan/repository.go` | CRUD for connections + sources; `AdvanceMarker` | Rewrite |
|
||||
| `internal/autoscan/provider.go` | `ScanSourceProvider` interface + adapter over the plugins resolver | Create |
|
||||
| `internal/autoscan/connection.go` | Resolve a `Connection` (own or Requests-linked) → `{base_url, api_key}` | Create |
|
||||
| `internal/autoscan/service.go` | Generic engine: per source → `PollChanges(marker)` → resolve → suppress → enqueue → store marker | Rewrite |
|
||||
| `internal/autoscan/dedupe.go`, `suppress.go` | Unchanged salvage | Keep |
|
||||
| `internal/autoscan/history.go`, `rewrite.go`, `suggest*.go` | Arr-specific | Delete |
|
||||
| `internal/taskmanager/tasks/autoscan_poll.go` | Poll task → `Service.PollOnce` | Keep (retarget) |
|
||||
| `internal/api/handlers/autoscan.go` | Admin API (settings/connections/sources/trigger/status) | Rewrite |
|
||||
| `internal/api/router.go`, `cmd/silo/main.go` | Wire the rebuilt service + connection resolver; drop request-coupled wiring | Modify |
|
||||
| `go.mod` | SDK replace (dev) → `v0.5.0` (final) | Modify |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Host `pluginhost` client wrapper for `scan_source.v1`
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/pluginhost/client.go`
|
||||
- Modify: `internal/pluginhost/client_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
In `internal/pluginhost/client_test.go`, add a test mirroring the existing `scheduled_task` capability-gate test (find it first; it constructs a `Client` with a manifest declaring a capability and asserts the accessor errors when the capability is absent and succeeds when present):
|
||||
|
||||
```go
|
||||
func TestClientScanSourceRequiresCapability(t *testing.T) {
|
||||
// Client whose manifest does NOT declare scan_source.v1
|
||||
c := newTestClientWithCapabilities(t, "metadata_provider.v1")
|
||||
if _, err := c.ScanSource("missing"); err == nil {
|
||||
t.Fatal("ScanSource: expected error when capability absent")
|
||||
}
|
||||
// Client WHOSE manifest declares scan_source.v1 with id "arr"
|
||||
c2 := newTestClientWithCapabilities(t, "scan_source.v1:arr")
|
||||
if _, err := c2.ScanSource("arr"); err != nil {
|
||||
t.Fatalf("ScanSource: unexpected error when capability present: %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use whatever helper the sibling `scheduled_task` test uses to construct a `Client` with declared capabilities; mirror it exactly (do not invent a helper that does not exist — read `client_test.go` and copy the construction).
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `go test ./internal/pluginhost/ -run TestClientScanSourceRequiresCapability`
|
||||
Expected: FAIL — `c.ScanSource undefined`.
|
||||
|
||||
- [ ] **Step 3: Add the wrapper type and accessor**
|
||||
|
||||
In `internal/pluginhost/client.go`, after the `ScheduledTaskClient` type (l.42), add:
|
||||
|
||||
```go
|
||||
type ScanSourceClient struct {
|
||||
client pluginv1.ScanSourceClient
|
||||
timeout time.Duration
|
||||
}
|
||||
```
|
||||
|
||||
After the `Client.ScheduledTask` accessor (l.117), add:
|
||||
|
||||
```go
|
||||
func (c *Client) ScanSource(capabilityID string) (*ScanSourceClient, error) {
|
||||
if err := c.requireCapability("scan_source.v1", capabilityID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ScanSourceClient{
|
||||
client: c.rpc.ScanSource(),
|
||||
timeout: DefaultControlTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ScanSourceClient) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.PollChanges(callCtx, req)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `go test ./internal/pluginhost/ -run TestClientScanSourceRequiresCapability`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/pluginhost/client.go internal/pluginhost/client_test.go
|
||||
git commit -m "feat(pluginhost): scan_source.v1 capability client wrapper"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `plugins.Service` resolver for `scan_source.v1`
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/plugins/service.go`
|
||||
|
||||
- [ ] **Step 1: Add the resolver method**
|
||||
|
||||
After `ScheduledTaskClient` (l.413–423), add the identical shape for scan source:
|
||||
|
||||
```go
|
||||
func (s *Service) ScanSourceClient(
|
||||
ctx context.Context,
|
||||
installationID int,
|
||||
capabilityID string,
|
||||
) (*pluginhost.ScanSourceClient, error) {
|
||||
client, err := s.ensureClient(ctx, installationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.ScanSource(capabilityID)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Build to verify it compiles**
|
||||
|
||||
Run: `go build ./internal/plugins/`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/plugins/service.go
|
||||
git commit -m "feat(plugins): expose scan_source.v1 client resolver"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Schema — `autoscan_connections` + `autoscan_sources`
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/172_autoscan_v2.up.sql`, `migrations/172_autoscan_v2.down.sql`
|
||||
|
||||
- [ ] **Step 1: Write the up migration**
|
||||
|
||||
Create `migrations/172_autoscan_v2.up.sql`. It drops PR #43's `171` tables and creates the decoupled schema (no FK into `request_integrations`; the optional reuse link is `ON DELETE SET NULL`):
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS public.autoscan_sources;
|
||||
DROP TABLE IF EXISTS public.autoscan_settings;
|
||||
|
||||
CREATE TABLE public.autoscan_settings (
|
||||
id boolean PRIMARY KEY DEFAULT true,
|
||||
enabled boolean NOT NULL DEFAULT false,
|
||||
default_poll_interval_seconds integer NOT NULL DEFAULT 600,
|
||||
debounce_seconds integer NOT NULL DEFAULT 60,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT autoscan_settings_singleton CHECK (id),
|
||||
CONSTRAINT autoscan_settings_interval_pos CHECK (default_poll_interval_seconds > 0),
|
||||
CONSTRAINT autoscan_settings_debounce_nonneg CHECK (debounce_seconds >= 0)
|
||||
);
|
||||
INSERT INTO public.autoscan_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
CREATE TABLE public.autoscan_connections (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name text NOT NULL,
|
||||
kind text NOT NULL,
|
||||
-- Own credentials (used when request_integration_id IS NULL):
|
||||
base_url text,
|
||||
api_key_ref text,
|
||||
-- Optional soft reuse of a Requests arr server (live link; SET NULL on delete):
|
||||
request_integration_id text REFERENCES public.request_integrations(id) ON DELETE SET NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT autoscan_connections_source_present
|
||||
CHECK (request_integration_id IS NOT NULL OR base_url IS NOT NULL)
|
||||
);
|
||||
|
||||
CREATE TABLE public.autoscan_sources (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
installation_id integer NOT NULL,
|
||||
capability_id text NOT NULL,
|
||||
connection_id uuid NOT NULL REFERENCES public.autoscan_connections(id) ON DELETE RESTRICT,
|
||||
enabled boolean NOT NULL DEFAULT false,
|
||||
poll_interval_seconds integer,
|
||||
marker text,
|
||||
last_run_at timestamptz,
|
||||
last_error text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT autoscan_sources_interval_pos CHECK (poll_interval_seconds IS NULL OR poll_interval_seconds > 0),
|
||||
CONSTRAINT autoscan_sources_capability_unique UNIQUE (installation_id, capability_id)
|
||||
);
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write the down migration**
|
||||
|
||||
Create `migrations/172_autoscan_v2.down.sql`:
|
||||
|
||||
```sql
|
||||
DROP TABLE IF EXISTS public.autoscan_sources;
|
||||
DROP TABLE IF EXISTS public.autoscan_connections;
|
||||
DROP TABLE IF EXISTS public.autoscan_settings;
|
||||
```
|
||||
|
||||
(Recreating the `171` tables on down-migration is unnecessary: `171` is unreleased and superseded; the down simply removes `172`'s tables.)
|
||||
|
||||
- [ ] **Step 3: Apply and verify against a disposable DB**
|
||||
|
||||
Run the migrations against a throwaway Postgres (the project's disposable-DB pattern) and confirm boot applies cleanly:
|
||||
```bash
|
||||
docker compose up -d postgres
|
||||
go run ./cmd/silo --migrate-only 2>&1 | grep -i "migrations applied" || echo "check migration output"
|
||||
```
|
||||
Expected: migrations apply without error; `\d autoscan_connections` shows the soft FK `request_integration_id`.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add migrations/172_autoscan_v2.up.sql migrations/172_autoscan_v2.down.sql
|
||||
git commit -m "feat(migrations): autoscan v2 schema (connections + sources, no requests FK)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Types and repository
|
||||
|
||||
**Files:**
|
||||
- Rewrite: `internal/autoscan/types.go`
|
||||
- Rewrite: `internal/autoscan/repository.go`
|
||||
|
||||
- [ ] **Step 1: Define the types**
|
||||
|
||||
Replace `internal/autoscan/types.go` with the v2 model:
|
||||
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import "time"
|
||||
|
||||
type Settings struct {
|
||||
Enabled bool
|
||||
DefaultPollIntervalSeconds int
|
||||
DebounceSeconds int
|
||||
}
|
||||
|
||||
// Connection is an arr server the host can reach: either own credentials, or a
|
||||
// live reference to a Requests integration (RequestIntegrationID set).
|
||||
type Connection struct {
|
||||
ID string
|
||||
Name string
|
||||
Kind string
|
||||
BaseURL string // own; empty when linked
|
||||
APIKeyRef string // own; empty when linked
|
||||
RequestIntegrationID *string
|
||||
}
|
||||
|
||||
// Source ties a scan_source plugin capability instance to a connection plus the
|
||||
// host-owned scheduling/bookkeeping state.
|
||||
type Source struct {
|
||||
ID string
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
ConnectionID string
|
||||
Enabled bool
|
||||
PollIntervalSeconds *int // nil => use settings default
|
||||
Marker *string // opaque; nil on first run
|
||||
LastRunAt *time.Time
|
||||
LastError *string
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write a failing repository test**
|
||||
|
||||
Add `internal/autoscan/repository_test.go` exercising round-trip + `AdvanceMarker` against a disposable DB (gate with the project's DB-test build tag/helper if one exists; otherwise a `//go:build integration` tag matching sibling DB tests). Minimal:
|
||||
|
||||
```go
|
||||
func TestRepositoryMarkerRoundTrip(t *testing.T) {
|
||||
repo := newTestRepo(t) // disposable-DB helper used by sibling repo tests
|
||||
connID := repo.mustCreateConnection(t, Connection{Name: "Sonarr", Kind: "sonarr", BaseURL: "http://x", APIKeyRef: "k"})
|
||||
srcID := repo.mustCreateSource(t, Source{InstallationID: 1, CapabilityID: "arr", ConnectionID: connID, Enabled: true})
|
||||
|
||||
if err := repo.AdvanceMarker(ctx, srcID, "2026-06-02T14:10:00Z"); err != nil {
|
||||
t.Fatalf("AdvanceMarker: %v", err)
|
||||
}
|
||||
got, _ := repo.ListEnabledSources(ctx)
|
||||
if len(got) != 1 || got[0].Marker == nil || *got[0].Marker != "2026-06-02T14:10:00Z" {
|
||||
t.Fatalf("marker not persisted: %+v", got)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run to verify it fails**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestRepositoryMarkerRoundTrip` (with the DB helper available)
|
||||
Expected: FAIL — undefined repository methods.
|
||||
|
||||
- [ ] **Step 4: Implement the repository**
|
||||
|
||||
Rewrite `internal/autoscan/repository.go` with: `GetSettings`, `UpdateSettings`, connection CRUD (`CreateConnection`, `UpdateConnection`, `DeleteConnection`, `ListConnections`, `GetConnection`), source CRUD (`UpsertSource`, `ListEnabledSources`, `ListSources`, `GetSource`), and `AdvanceMarker(ctx, sourceID, marker string)` (sets `marker`, `last_run_at = now()`, clears `last_error`). Use pgx; follow the column/scan style of the existing (pre-rewrite) `repository.go` and `internal/requests/repository.go`.
|
||||
|
||||
- [ ] **Step 5: Run to verify it passes**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestRepositoryMarkerRoundTrip`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/types.go internal/autoscan/repository.go internal/autoscan/repository_test.go
|
||||
git commit -m "feat(autoscan): v2 types and repository (connections, sources, markers)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Connection resolution (own or Requests-linked)
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/connection.go`
|
||||
- Create: `internal/autoscan/connection_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```go
|
||||
func TestResolveConnectionOwnVsLinked(t *testing.T) {
|
||||
reqLookup := fakeRequestIntegrationLookup{"req-1": creds{BaseURL: "http://req:7878", APIKeyRef: "rk"}}
|
||||
secrets := fakeSecrets{"rk": "REQKEY", "ownref": "OWNKEY"}
|
||||
r := NewConnectionResolver(reqLookup, secrets)
|
||||
|
||||
own, err := r.Resolve(ctx, Connection{BaseURL: "http://own:8989", APIKeyRef: "ownref"})
|
||||
if err != nil || own.BaseURL != "http://own:8989" || own.APIKey != "OWNKEY" {
|
||||
t.Fatalf("own resolve = %+v, err=%v", own, err)
|
||||
}
|
||||
id := "req-1"
|
||||
linked, err := r.Resolve(ctx, Connection{RequestIntegrationID: &id})
|
||||
if err != nil || linked.BaseURL != "http://req:7878" || linked.APIKey != "REQKEY" {
|
||||
t.Fatalf("linked resolve = %+v, err=%v", linked, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveConnectionLinkedMissingFallsBack(t *testing.T) {
|
||||
r := NewConnectionResolver(fakeRequestIntegrationLookup{}, fakeSecrets{})
|
||||
id := "gone"
|
||||
if _, err := r.Resolve(ctx, Connection{RequestIntegrationID: &id}); err == nil {
|
||||
t.Fatal("expected error when linked Requests integration is missing")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestResolveConnection`
|
||||
Expected: FAIL — undefined `NewConnectionResolver`.
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
Create `internal/autoscan/connection.go`:
|
||||
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// ResolvedConnection is concrete credentials handed to the plugin.
|
||||
type ResolvedConnection struct {
|
||||
BaseURL string
|
||||
APIKey string
|
||||
}
|
||||
|
||||
type RequestIntegrationLookup interface {
|
||||
// Returns base URL + encrypted api key ref for a Requests integration.
|
||||
Get(ctx context.Context, integrationID string) (baseURL, apiKeyRef string, err error)
|
||||
}
|
||||
|
||||
type SecretResolver interface {
|
||||
Get(ctx context.Context, ref string) (string, error)
|
||||
}
|
||||
|
||||
type ConnectionResolver struct {
|
||||
requests RequestIntegrationLookup
|
||||
secrets SecretResolver
|
||||
}
|
||||
|
||||
func NewConnectionResolver(r RequestIntegrationLookup, s SecretResolver) *ConnectionResolver {
|
||||
return &ConnectionResolver{requests: r, secrets: s}
|
||||
}
|
||||
|
||||
func (cr *ConnectionResolver) Resolve(ctx context.Context, c Connection) (ResolvedConnection, error) {
|
||||
baseURL, apiKeyRef := c.BaseURL, c.APIKeyRef
|
||||
if c.RequestIntegrationID != nil {
|
||||
u, ref, err := cr.requests.Get(ctx, *c.RequestIntegrationID)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: linked requests integration %q: %w", *c.RequestIntegrationID, err)
|
||||
}
|
||||
baseURL, apiKeyRef = u, ref
|
||||
}
|
||||
apiKey := apiKeyRef
|
||||
if cr.secrets != nil && apiKeyRef != "" {
|
||||
resolved, err := cr.secrets.Get(ctx, apiKeyRef)
|
||||
if err != nil {
|
||||
return ResolvedConnection{}, fmt.Errorf("autoscan: resolve api key: %w", err)
|
||||
}
|
||||
if resolved != "" {
|
||||
apiKey = resolved
|
||||
}
|
||||
}
|
||||
return ResolvedConnection{BaseURL: baseURL, APIKey: apiKey}, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestResolveConnection`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/connection.go internal/autoscan/connection_test.go
|
||||
git commit -m "feat(autoscan): resolve connections (own credentials or Requests-linked)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Provider seam over the plugin resolver
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/provider.go`
|
||||
|
||||
- [ ] **Step 1: Define the seam and adapter**
|
||||
|
||||
The engine must be testable without a live plugin, so it depends on a narrow interface, with a production adapter over `plugins.Service.ScanSourceClient`. Create `internal/autoscan/provider.go`:
|
||||
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
)
|
||||
|
||||
// ScanSourceProvider yields changed paths for one source. The engine calls
|
||||
// PollChanges; production wraps the plugins.Service scan_source resolver.
|
||||
type ScanSourceProvider interface {
|
||||
PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection) (paths []string, nextMarker string, err error)
|
||||
}
|
||||
|
||||
// pluginScanSourceClient is the slice of *pluginhost.ScanSourceClient used here.
|
||||
type pluginScanSourceClient interface {
|
||||
PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error)
|
||||
}
|
||||
|
||||
type scanSourceResolver interface {
|
||||
ScanSourceClient(ctx context.Context, installationID int, capabilityID string) (pluginScanSourceClient, error)
|
||||
}
|
||||
|
||||
type pluginProvider struct{ resolver scanSourceResolver }
|
||||
|
||||
func NewPluginProvider(resolver scanSourceResolver) ScanSourceProvider {
|
||||
return &pluginProvider{resolver: resolver}
|
||||
}
|
||||
|
||||
func (p *pluginProvider) PollChanges(ctx context.Context, installationID int, capabilityID, marker string, conn ResolvedConnection) ([]string, string, error) {
|
||||
client, err := p.resolver.ScanSourceClient(ctx, installationID, capabilityID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
resp, err := client.PollChanges(ctx, &pluginv1.PollChangesRequest{CapabilityId: capabilityID, Marker: marker})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return resp.GetChangedPaths(), resp.GetNextMarker(), nil
|
||||
}
|
||||
```
|
||||
|
||||
Note: the connection (`conn`) is delivered to the plugin out-of-band as runtime config when the source is configured (the plugin reads its own `{base_url, api_key}`); it is passed here so a future provider variant could inject per-call. For v1 the production path configures the plugin instance with the resolved connection at upsert time; document this in `NewPluginProvider`'s comment.
|
||||
|
||||
- [ ] **Step 2: Build**
|
||||
|
||||
Run: `go build ./internal/autoscan/`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/provider.go
|
||||
git commit -m "feat(autoscan): scan-source provider seam over the plugin resolver"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Generalize the engine (`Service.PollOnce`)
|
||||
|
||||
**Files:**
|
||||
- Rewrite: `internal/autoscan/service.go`
|
||||
- Modify: `internal/autoscan/service_test.go` (generalize the salvaged tests)
|
||||
|
||||
- [ ] **Step 1: Adapt the salvaged tests to the provider seam**
|
||||
|
||||
The PR #43 `service_test.go` used a `fakeHistory`; replace it with a `fakeProvider` implementing `ScanSourceProvider`, keeping the same assertions (dedupe to one folder; distinct paths same folder → two scans; disabled no-op; provider error keeps marker; enqueue failure releases claims + keeps marker). Add an opaque-marker assertion: the provider's returned `nextMarker` is stored verbatim via `AdvanceMarker`.
|
||||
|
||||
```go
|
||||
type fakeProvider struct {
|
||||
paths map[string][]string // key: capabilityID
|
||||
nextMarker string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeProvider) PollChanges(_ context.Context, _ int, capabilityID, _ string, _ ResolvedConnection) ([]string, string, error) {
|
||||
if f.err != nil {
|
||||
return nil, "", f.err
|
||||
}
|
||||
return f.paths[capabilityID], f.nextMarker, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify the tests fail**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestPollOnce`
|
||||
Expected: FAIL — `Service` signature/`fakeProvider` not yet wired.
|
||||
|
||||
- [ ] **Step 3: Rewrite the engine**
|
||||
|
||||
Rewrite `internal/autoscan/service.go`. Keep the salvaged resolve→suppress→enqueue loop verbatim (it is provider-agnostic already), but drive it from sources + the provider seam:
|
||||
|
||||
```go
|
||||
func (s *Service) PollOnce(ctx context.Context) error {
|
||||
settings, err := s.store.GetSettings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !settings.Enabled {
|
||||
return nil
|
||||
}
|
||||
sources, err := s.store.ListEnabledSources(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ttl := time.Duration(settings.DebounceSeconds) * time.Second
|
||||
|
||||
for _, src := range sources {
|
||||
conn, cerr := s.resolveConnection(ctx, src.ConnectionID)
|
||||
if cerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: resolve connection failed", "source_id", src.ID, "err", cerr)
|
||||
continue
|
||||
}
|
||||
marker := ""
|
||||
if src.Marker != nil {
|
||||
marker = *src.Marker
|
||||
}
|
||||
paths, next, perr := s.provider.PollChanges(ctx, src.InstallationID, src.CapabilityID, marker, conn)
|
||||
if perr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: poll changes failed", "source_id", src.ID, "err", perr)
|
||||
_ = s.store.RecordError(ctx, src.ID, perr.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
targets, claimed := s.resolveAndClaim(ctx, paths, ttl) // salvaged dedupe→resolve→suppress
|
||||
if len(targets) > 0 {
|
||||
if eerr := s.queue.EnqueueScans(ctx, targets); eerr != nil {
|
||||
s.releaseClaims(ctx, claimed)
|
||||
slog.WarnContext(ctx, "autoscan: enqueue failed", "source_id", src.ID, "err", eerr)
|
||||
continue // do NOT advance marker
|
||||
}
|
||||
}
|
||||
if aerr := s.store.AdvanceMarker(ctx, src.ID, next); aerr != nil {
|
||||
slog.WarnContext(ctx, "autoscan: advance marker failed", "source_id", src.ID, "err", aerr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
Extract the salvaged loop body into `resolveAndClaim`/`releaseClaims` helpers (lifted verbatim from PR #43's `service.go` lines that build `targets`/`claimed`). Keep `uniqueParentDirs`, `Suppressor`, the `fmt.Sprintf("%d|%s", folderID, target.Path)` suppression key, and the `scantrigger.RequestError` quiet-skip.
|
||||
|
||||
- [ ] **Step 4: Run to verify the tests pass**
|
||||
|
||||
Run: `go test ./internal/autoscan/ -run TestPollOnce`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/autoscan/service.go internal/autoscan/service_test.go
|
||||
git commit -m "feat(autoscan): generic engine drives sources via scan_source provider"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Delete arr-specific salvage residue
|
||||
|
||||
**Files:**
|
||||
- Delete: `internal/autoscan/history.go`, `history_test.go`, `rewrite.go`, `rewrite_test.go`, `suggest.go`, `suggest_deps.go`, `suggest_test.go`
|
||||
|
||||
- [ ] **Step 1: Remove the files**
|
||||
|
||||
```bash
|
||||
git rm internal/autoscan/history.go internal/autoscan/history_test.go \
|
||||
internal/autoscan/rewrite.go internal/autoscan/rewrite_test.go \
|
||||
internal/autoscan/suggest.go internal/autoscan/suggest_deps.go internal/autoscan/suggest_test.go
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Build to confirm no remaining references**
|
||||
|
||||
Run: `go build ./internal/autoscan/...`
|
||||
Expected: exits 0 (the engine no longer references the deleted arr logic). If it fails, a reference was missed — remove it.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git commit -m "refactor(autoscan): drop arr-specific history/rewrite/suggest (moved to plugin)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Admin API rewrite
|
||||
|
||||
**Files:**
|
||||
- Rewrite: `internal/api/handlers/autoscan.go`
|
||||
- Modify: `internal/api/handlers/autoscan_test.go`
|
||||
|
||||
- [ ] **Step 1: Define the endpoints**
|
||||
|
||||
Rewrite the handler to back the new model. Endpoints (admin-gated, mounted in Task 10):
|
||||
- `GET/PUT /admin/autoscan/settings`
|
||||
- `GET/POST /admin/autoscan/connections`, `PUT/DELETE /admin/autoscan/connections/{id}`
|
||||
- `GET /admin/autoscan/sources`, `PUT /admin/autoscan/sources/{id}` (enable, interval, connection binding)
|
||||
- `POST /admin/autoscan/trigger` (async `PollOnce`, 202 — keep the detached-goroutine + channel-tested pattern from the closed PR's `autoscan_test.go`)
|
||||
- `GET /admin/autoscan/status`
|
||||
|
||||
Strip credentials from responses (never emit `api_key_ref`/resolved keys), matching the `_SENSITIVE_METADATA_KEYS` defense-in-depth posture.
|
||||
|
||||
- [ ] **Step 2: Port + extend the handler tests**
|
||||
|
||||
Carry over the trigger test (the `done`-channel synchronization fix from `d3ee373`), the FK→404 test (now: unknown source/connection → 404), and a secrets-not-leaked test for the connections list.
|
||||
|
||||
- [ ] **Step 3: Run the handler tests (libvips container)**
|
||||
|
||||
Run (in the libvips-equipped container per Prerequisites #3):
|
||||
`go test ./internal/api/handlers/ -run Autoscan`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/api/handlers/autoscan.go internal/api/handlers/autoscan_test.go
|
||||
git commit -m "feat(api): autoscan v2 admin endpoints (settings, connections, sources)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 10: Wiring + rate limits + retire old coupling
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/api/router.go`
|
||||
- Modify: `cmd/silo/main.go`
|
||||
|
||||
- [ ] **Step 1: Rebuild the service wiring**
|
||||
|
||||
In `internal/api/router.go` and `cmd/silo/main.go`, construct the v2 service:
|
||||
|
||||
```go
|
||||
autoscanRepo := autoscan.NewRepository(deps.DB)
|
||||
provider := autoscan.NewPluginProvider(pluginServiceScanSourceAdapter{pluginService})
|
||||
connResolver := autoscan.NewConnectionResolver(requestIntegrationLookup{requestsRepo}, serverSettingsSecretResolver{settingsRepo})
|
||||
autoscanSvc := autoscan.NewService(autoscanRepo, provider, connResolver, deps.Resolver, deps.ScanQueue, autoscan.NewRedisSuppressor(deps.RedisClient))
|
||||
autoscanHandler = handlers.NewAutoscanHandler(autoscanRepo, autoscanSvc)
|
||||
```
|
||||
|
||||
Replace the old route block with the Task 9 routes. Keep the `catalog_*`/admin rate-limit conventions consistent with the security standards (per-user/admin caps on the new endpoints; mirror the request-integrations admin tier).
|
||||
|
||||
- [ ] **Step 2: Retarget the poll task**
|
||||
|
||||
`internal/taskmanager/tasks/autoscan_poll.go` calls `Service.PollOnce` — unchanged behavior; confirm it compiles against the rebuilt service and uses `Settings.DefaultPollIntervalSeconds` for the interval (per-source overrides are honored inside `PollOnce`).
|
||||
|
||||
- [ ] **Step 3: Build the whole tree**
|
||||
|
||||
Run (libvips container): `go build ./...`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/api/router.go cmd/silo/main.go internal/taskmanager/tasks/autoscan_poll.go
|
||||
git commit -m "feat(autoscan): wire v2 service, routes, and poll task"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 11: Full sweep + finalize SDK dependency
|
||||
|
||||
**Files:**
|
||||
- Modify: `go.mod`, `go.sum`
|
||||
|
||||
- [ ] **Step 1: Full build + test (libvips container)**
|
||||
|
||||
Run in the libvips-equipped container (Prerequisites #3):
|
||||
```bash
|
||||
go build ./... && go test ./... && go vet ./...
|
||||
```
|
||||
Expected: build OK; all packages `ok`; vet clean. `gofmt -l internal migrations` prints nothing.
|
||||
|
||||
- [ ] **Step 2: Finalize the SDK version (after `v0.5.0` is tagged upstream)**
|
||||
|
||||
Once `silo-plugin-sdk` PR #2 is merged and `v0.5.0` tagged:
|
||||
```bash
|
||||
go mod edit -dropreplace github.com/Silo-Server/silo-plugin-sdk
|
||||
go get github.com/Silo-Server/silo-plugin-sdk@v0.5.0
|
||||
go mod tidy
|
||||
go build ./...
|
||||
```
|
||||
Expected: builds against the released tag with no replace directive.
|
||||
|
||||
If `v0.5.0` is not yet tagged when this plan is otherwise complete, leave the replace directive in place, commit it, and note in the PR that the dependency must be finalized before merge (do not merge a `replace` pointing at a local path).
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add go.mod go.sum
|
||||
git commit -m "build: depend on silo-plugin-sdk v0.5.0 (scan_source.v1)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** §7 engine (Tasks 6–7), §8 connections incl. reuse-or-own + soft Requests link (Tasks 3,5), §9 data model decoupled from Requests + Autoscan-owned state (Tasks 3–4,9–10), §5/§11 host consumes `scan_source.v1` (Tasks 1–2). §10 (arr plugin) and §9 UI are separate plans.
|
||||
- **Salvage map honored (§13):** engine/dedupe/suppress kept and generalized (Tasks 6–8); arr history/rewrite/suggest deleted (Task 8); `last_poll_at` → opaque `marker` (Tasks 3–4,7).
|
||||
- **Out of scope here:** the Autoscan admin **UI** category (Part 2 plan) and the **arr plugin** (separate repo plan). This backend is testable on its own via the repository/engine/handler tests.
|
||||
- **Risk carried from spec §14:** plugin multiplicity (one installation per arr server vs. many connections per installation) is modeled as one `autoscan_sources` row per `(installation_id, capability_id)`; if the runtime supports multiple capability instances per installation this maps cleanly, otherwise it implies one installation per server. Confirm during Task 1–2 by inspecting `ensureClient`/installation handling.
|
||||
- **Type consistency:** `ResolvedConnection{BaseURL,APIKey}`, `Connection{BaseURL,APIKeyRef,RequestIntegrationID}`, `Source{InstallationID,CapabilityID,ConnectionID,Marker}`, `ScanSourceProvider.PollChanges(...)`, `Service.PollOnce` used consistently across Tasks 4–10.
|
||||
@@ -0,0 +1,694 @@
|
||||
# Autoscan Rewrite-Sync Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** A "Sync rewrites from arr" action that suggests autoscan `path_rewrites` for an instance by matching its arr root folders to Silo media folders on shared trailing path segments, previewed and committed by the admin.
|
||||
|
||||
**Architecture:** A pure `suggestRewrites` matcher in `internal/autoscan`, a `Service.SuggestRewrites` method backed by two new injected deps (an arr root-folder client and a Silo folder lister), one read-only admin endpoint returning a preview, and a "Sync from arr" button in the existing Autoscan tab that merges proposals into the editor (admin saves via the existing per-source PUT). Manual rewrites stay the source of truth.
|
||||
|
||||
**Tech Stack:** Go (standard `testing`), React/TypeScript.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-06-02-autoscan-rewrite-sync-design.md`
|
||||
|
||||
**Commands assume the repository root is the cwd.** Go tests: `go test ./internal/autoscan/...`. Frontend: `cd web && pnpm exec tsc -b && pnpm run lint`. Ensure the Go toolchain is on `PATH` (prepend its `bin` directory if `go` is missing). `internal/api`/`internal/api/handlers` cannot be compiled without libvips; verify those via `gofmt` + the Docker build.
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 — Branch
|
||||
|
||||
- [ ] **Step 0.1: Confirm branch**
|
||||
|
||||
```bash
|
||||
git rev-parse --abbrev-ref HEAD # expect: feat/autoscan-arr-polling
|
||||
```
|
||||
This feature extends the autoscan branch. If you're elsewhere: `git checkout feat/autoscan-arr-polling`.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Core matcher (pure, TDD)
|
||||
|
||||
### Task 1: suggestRewrites + types
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/suggest.go`
|
||||
- Create: `internal/autoscan/suggest_test.go`
|
||||
|
||||
- [ ] **Step 1.1: Write the failing test**
|
||||
|
||||
`internal/autoscan/suggest_test.go`:
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSuggestRewrites(t *testing.T) {
|
||||
silo := []string{
|
||||
"/mnt/media/happy/storage2/tvshows1",
|
||||
"/mnt/media/happy4k/4ktv7",
|
||||
"/mnt/media/storage/Anime/Subs",
|
||||
"/mnt/media/storage/Anime2/Subs",
|
||||
"/library/Films",
|
||||
"/tank/television/Show",
|
||||
}
|
||||
|
||||
t.Run("multi-segment unique match", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/mnt/happy/storage2/tvshows1"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" || got.Proposed[0].MatchDepth != 2 {
|
||||
t.Fatalf("proposed=%+v", got.Proposed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("single-segment unique match (different parents)", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/mnt/kodama/storage2/4ktv7"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy4k/4ktv7" || got.Proposed[0].MatchDepth != 1 {
|
||||
t.Fatalf("proposed=%+v", got.Proposed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("longest-suffix disambiguation", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/mnt/kodama/storage1/Anime/Subs"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/storage/Anime/Subs" || got.Proposed[0].MatchDepth != 2 {
|
||||
t.Fatalf("expected Anime/Subs (depth 2), got %+v", got.Proposed)
|
||||
}
|
||||
if len(got.Ambiguous) != 0 {
|
||||
t.Fatalf("should not be ambiguous: %+v", got.Ambiguous)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unmatched when no shared segment", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/data/Movies"}, silo, nil)
|
||||
if len(got.Unmatched) != 1 || got.Unmatched[0] != "/data/Movies" || len(got.Proposed) != 0 {
|
||||
t.Fatalf("got=%+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("leaf match across unlike layouts", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{"/srv/tv/Show"}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/tank/television/Show" {
|
||||
t.Fatalf("got=%+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ambiguous tie", func(t *testing.T) {
|
||||
// a root whose only shared segment is "Subs" ties two folders
|
||||
got := suggestRewrites([]string{"/foo/bar/Subs"}, silo, nil)
|
||||
if len(got.Ambiguous) != 1 || len(got.Ambiguous[0].Candidates) != 2 {
|
||||
t.Fatalf("expected ambiguous with 2 candidates, got %+v", got.Ambiguous)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("covered by existing rule", func(t *testing.T) {
|
||||
existing := []PathRewrite{{From: "/mnt/happy", To: "/mnt/media/happy"}}
|
||||
got := suggestRewrites([]string{"/mnt/happy/storage2/tvshows1"}, silo, existing)
|
||||
if len(got.Covered) != 1 || len(got.Proposed) != 0 {
|
||||
t.Fatalf("expected covered, got %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normalization: trailing slash, backslashes, dup slashes", func(t *testing.T) {
|
||||
got := suggestRewrites([]string{`\mnt\happy\\storage2\tvshows1\`}, silo, nil)
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].From != "/mnt/happy/storage2/tvshows1" || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" {
|
||||
t.Fatalf("normalization failed: %+v", got.Proposed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCommonSuffixLen(t *testing.T) {
|
||||
cases := []struct {
|
||||
a, b []string
|
||||
want int
|
||||
}{
|
||||
{[]string{"a", "b", "c"}, []string{"x", "b", "c"}, 2},
|
||||
{[]string{"4ktv7"}, []string{"happy4k", "4ktv7"}, 1},
|
||||
{[]string{"4ktv7"}, []string{"4ktv70"}, 0}, // full-segment, not substring
|
||||
{[]string{"a"}, []string{"b"}, 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := commonSuffixLen(tc.a, tc.b); got != tc.want {
|
||||
t.Fatalf("commonSuffixLen(%v,%v)=%d want %d", tc.a, tc.b, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 1.2: Run it — expect failure**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/ -run 'TestSuggestRewrites|TestCommonSuffixLen' -v
|
||||
```
|
||||
Expected: FAIL (`suggestRewrites` undefined).
|
||||
|
||||
- [ ] **Step 1.3: Implement** `internal/autoscan/suggest.go`:
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import "strings"
|
||||
|
||||
// RewriteSuggestions is the result of matching arr root folders to Silo folders.
|
||||
type RewriteSuggestions struct {
|
||||
Proposed []ProposedRewrite `json:"proposed"`
|
||||
Unmatched []string `json:"unmatched"`
|
||||
Ambiguous []AmbiguousRoot `json:"ambiguous"`
|
||||
Covered []string `json:"covered"`
|
||||
}
|
||||
|
||||
// ProposedRewrite is a suggested rewrite plus its confidence (shared trailing segments).
|
||||
type ProposedRewrite struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
MatchDepth int `json:"match_depth"`
|
||||
}
|
||||
|
||||
// AmbiguousRoot is an arr root that tied across multiple Silo folders.
|
||||
type AmbiguousRoot struct {
|
||||
Root string `json:"root"`
|
||||
Candidates []string `json:"candidates"`
|
||||
}
|
||||
|
||||
// normalizePath makes a path comparable: backslashes -> '/', collapse duplicate
|
||||
// slashes, strip a trailing slash (but keep a bare "/").
|
||||
func normalizePath(p string) string {
|
||||
p = normalizeSeparators(strings.TrimSpace(p))
|
||||
for strings.Contains(p, "//") {
|
||||
p = strings.ReplaceAll(p, "//", "/")
|
||||
}
|
||||
if len(p) > 1 {
|
||||
p = strings.TrimRight(p, "/")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func segments(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(p, "/")
|
||||
}
|
||||
|
||||
// commonSuffixLen counts equal trailing segments (full-segment equality).
|
||||
func commonSuffixLen(a, b []string) int {
|
||||
i, j, n := len(a)-1, len(b)-1, 0
|
||||
for i >= 0 && j >= 0 && a[i] == b[j] {
|
||||
n++
|
||||
i--
|
||||
j--
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// coveredBy reports whether an existing rewrite already matches root (same
|
||||
// boundary rule as applyRewrites).
|
||||
func coveredBy(root string, existing []PathRewrite) bool {
|
||||
for _, rw := range existing {
|
||||
from := strings.TrimRight(strings.TrimSpace(rw.From), "/")
|
||||
if from == "" {
|
||||
continue
|
||||
}
|
||||
if root == from || strings.HasPrefix(root, from+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// suggestRewrites matches each arr root to the Silo folder sharing the most
|
||||
// trailing path segments (unique winner). Roots already handled by an existing
|
||||
// rewrite are reported as Covered; roots with no shared segment as Unmatched;
|
||||
// ties as Ambiguous. Pure: no I/O, no deployment constants.
|
||||
func suggestRewrites(arrRoots, siloFolderPaths []string, existing []PathRewrite) RewriteSuggestions {
|
||||
siloNorm := make([]string, 0, len(siloFolderPaths))
|
||||
siloSegs := make([][]string, 0, len(siloFolderPaths))
|
||||
for _, p := range siloFolderPaths {
|
||||
n := normalizePath(p)
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
siloNorm = append(siloNorm, n)
|
||||
siloSegs = append(siloSegs, segments(n))
|
||||
}
|
||||
|
||||
var out RewriteSuggestions
|
||||
for _, raw := range arrRoots {
|
||||
root := normalizePath(raw)
|
||||
if root == "" {
|
||||
continue
|
||||
}
|
||||
if coveredBy(root, existing) {
|
||||
out.Covered = append(out.Covered, root)
|
||||
continue
|
||||
}
|
||||
rootSegs := segments(root)
|
||||
best := 0
|
||||
var winners []string
|
||||
for i, segs := range siloSegs {
|
||||
n := commonSuffixLen(rootSegs, segs)
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
if n > best {
|
||||
best, winners = n, []string{siloNorm[i]}
|
||||
} else if n == best {
|
||||
winners = append(winners, siloNorm[i])
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case best == 0:
|
||||
out.Unmatched = append(out.Unmatched, root)
|
||||
case len(winners) == 1:
|
||||
out.Proposed = append(out.Proposed, ProposedRewrite{From: root, To: winners[0], MatchDepth: best})
|
||||
default:
|
||||
out.Ambiguous = append(out.Ambiguous, AmbiguousRoot{Root: root, Candidates: winners})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 1.4: Run it — expect pass; commit**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/ -run 'TestSuggestRewrites|TestCommonSuffixLen' -v
|
||||
gofmt -l internal/autoscan/suggest.go internal/autoscan/suggest_test.go
|
||||
git add internal/autoscan/suggest.go internal/autoscan/suggest_test.go
|
||||
git commit -m "feat(autoscan): suffix-match rewrite suggester"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Service support
|
||||
|
||||
### Task 2: Repository.GetSource
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/autoscan/repository.go`
|
||||
|
||||
- [ ] **Step 2.1: Implement** `GetSource` (single-source join, includes disabled). Add after `ListEnabledSources`:
|
||||
```go
|
||||
// GetSource returns one instance's autoscan state joined with its
|
||||
// request_integrations row (kind/base_url/api_key_ref), regardless of enabled.
|
||||
func (r *Repository) GetSource(ctx context.Context, integrationID string) (*Source, error) {
|
||||
row := r.pool.QueryRow(ctx, sourceSelect+` WHERE ri.id = $1`, integrationID)
|
||||
src, err := scanSource(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrIntegrationNotFound, integrationID)
|
||||
}
|
||||
return nil, fmt.Errorf("get autoscan source: %w", err)
|
||||
}
|
||||
return &src, nil
|
||||
}
|
||||
```
|
||||
(`errors`, `pgx`, `fmt`, `ErrIntegrationNotFound`, and `sourceSelect`/`scanSource` already exist in the file.)
|
||||
|
||||
- [ ] **Step 2.2: Build + commit**
|
||||
|
||||
```bash
|
||||
go build ./internal/autoscan/... && go test ./internal/autoscan/...
|
||||
git add internal/autoscan/repository.go
|
||||
git commit -m "feat(autoscan): GetSource single-source lookup"
|
||||
```
|
||||
|
||||
### Task 3: RootFolderClient + FolderLister adapters
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/autoscan/suggest_deps.go`
|
||||
|
||||
- [ ] **Step 3.1: Implement the two adapters + interfaces**
|
||||
```go
|
||||
package autoscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/requests/arrclient"
|
||||
)
|
||||
|
||||
// RootFolderClient lists a Radarr/Sonarr instance's configured root folder paths.
|
||||
type RootFolderClient interface {
|
||||
RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error)
|
||||
}
|
||||
|
||||
// FolderLister lists every Silo media-folder path.
|
||||
type FolderLister interface {
|
||||
ListFolderPaths(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
type arrRootFolderClient struct{ httpClient *http.Client }
|
||||
|
||||
// NewArrRootFolderClient returns a RootFolderClient backed by the shared arrclient.
|
||||
func NewArrRootFolderClient(httpClient *http.Client) RootFolderClient {
|
||||
return &arrRootFolderClient{httpClient: httpClient}
|
||||
}
|
||||
|
||||
func (c *arrRootFolderClient) RootFolders(ctx context.Context, baseURL, apiKey string) ([]string, error) {
|
||||
client := arrclient.New(baseURL, apiKey, c.httpClient)
|
||||
folders, err := arrclient.ListRootFolders(ctx, client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
paths := make([]string, 0, len(folders))
|
||||
for _, f := range folders {
|
||||
if f.Path != "" {
|
||||
paths = append(paths, f.Path)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
type catalogFolderLister struct{ repo *catalog.FolderRepository }
|
||||
|
||||
// NewCatalogFolderLister adapts catalog.FolderRepository to FolderLister.
|
||||
func NewCatalogFolderLister(repo *catalog.FolderRepository) FolderLister {
|
||||
return &catalogFolderLister{repo: repo}
|
||||
}
|
||||
|
||||
func (l *catalogFolderLister) ListFolderPaths(ctx context.Context) ([]string, error) {
|
||||
folders, err := l.repo.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var paths []string
|
||||
for _, f := range folders {
|
||||
paths = append(paths, f.Paths...)
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
```
|
||||
Note: `autoscan` already depends on `catalog` (via `scantrigger`) and on `requests/arrclient` (via `history.go`), so these imports add no new module-level coupling and no cycle (`catalog` does not import `autoscan`).
|
||||
|
||||
- [ ] **Step 3.2: Build + commit**
|
||||
|
||||
```bash
|
||||
go build ./internal/autoscan/...
|
||||
git add internal/autoscan/suggest_deps.go
|
||||
git commit -m "feat(autoscan): arr root-folder client + Silo folder lister"
|
||||
```
|
||||
|
||||
### Task 4: Service.SuggestRewrites
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/autoscan/service.go`
|
||||
- Modify: `internal/autoscan/service_test.go`
|
||||
|
||||
- [ ] **Step 4.1: Write the failing test** (append to `service_test.go`):
|
||||
```go
|
||||
type fakeRootFolders struct {
|
||||
paths []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeRootFolders) RootFolders(context.Context, string, string) ([]string, error) {
|
||||
return f.paths, f.err
|
||||
}
|
||||
|
||||
type fakeFolderLister struct{ paths []string }
|
||||
|
||||
func (f fakeFolderLister) ListFolderPaths(context.Context) ([]string, error) { return f.paths, nil }
|
||||
|
||||
type sourceGetterStore struct {
|
||||
fakeStore
|
||||
src *Source
|
||||
}
|
||||
|
||||
func (s *sourceGetterStore) GetSource(context.Context, string) (*Source, error) { return s.src, nil }
|
||||
|
||||
func TestSuggestRewritesService(t *testing.T) {
|
||||
store := &sourceGetterStore{src: &Source{IntegrationID: "i1", Kind: "sonarr", BaseURL: "http://x", APIKeyRef: "k"}}
|
||||
svc := NewService(store, &fakeHistory{}, fakeResolver{}, &recordingQueuer{}, allowSuppressor{}, nil)
|
||||
svc.SetRewriteResolvers(
|
||||
fakeRootFolders{paths: []string{"/mnt/happy/storage2/tvshows1", "/data/Movies"}},
|
||||
fakeFolderLister{paths: []string{"/mnt/media/happy/storage2/tvshows1"}},
|
||||
)
|
||||
got, err := svc.SuggestRewrites(context.Background(), "i1")
|
||||
if err != nil {
|
||||
t.Fatalf("SuggestRewrites: %v", err)
|
||||
}
|
||||
if len(got.Proposed) != 1 || got.Proposed[0].To != "/mnt/media/happy/storage2/tvshows1" {
|
||||
t.Fatalf("proposed=%+v", got.Proposed)
|
||||
}
|
||||
if len(got.Unmatched) != 1 || got.Unmatched[0] != "/data/Movies" {
|
||||
t.Fatalf("unmatched=%+v", got.Unmatched)
|
||||
}
|
||||
}
|
||||
```
|
||||
> The `Store` interface needs a `GetSource` method so the service can call it. Add `GetSource(ctx context.Context, integrationID string) (*Source, error)` to the `Store` interface in `service.go`. `*Repository` already implements it (Task 2). The test's `sourceGetterStore` embeds the existing `fakeStore` and adds `GetSource`. If `fakeStore` does not satisfy the widened interface on its own, that's fine — only `sourceGetterStore` is used here; but ensure the package still compiles (other tests construct `fakeStore` directly and pass it where `Store` is required — add a `GetSource` method to `fakeStore` returning `(nil, nil)` so it still satisfies `Store`).
|
||||
|
||||
- [ ] **Step 4.2: Run it — expect failure**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/ -run TestSuggestRewritesService -v
|
||||
```
|
||||
Expected: FAIL (`SetRewriteResolvers`/`SuggestRewrites` undefined).
|
||||
|
||||
- [ ] **Step 4.3: Implement** in `service.go`. Add fields + setter + method, and add `GetSource` to `Store`:
|
||||
```go
|
||||
// add to the Store interface:
|
||||
// GetSource(ctx context.Context, integrationID string) (*Source, error)
|
||||
|
||||
// add fields to Service struct:
|
||||
// rootFolders RootFolderClient
|
||||
// folders FolderLister
|
||||
|
||||
// SetRewriteResolvers wires the deps used by SuggestRewrites (optional; only the
|
||||
// admin-facing service needs them).
|
||||
func (s *Service) SetRewriteResolvers(rootFolders RootFolderClient, folders FolderLister) {
|
||||
s.rootFolders = rootFolders
|
||||
s.folders = folders
|
||||
}
|
||||
|
||||
// SuggestRewrites matches an instance's arr root folders to Silo media folders.
|
||||
func (s *Service) SuggestRewrites(ctx context.Context, integrationID string) (RewriteSuggestions, error) {
|
||||
if s.rootFolders == nil || s.folders == nil {
|
||||
return RewriteSuggestions{}, fmt.Errorf("autoscan: rewrite suggestion not configured")
|
||||
}
|
||||
src, err := s.store.GetSource(ctx, integrationID)
|
||||
if err != nil {
|
||||
return RewriteSuggestions{}, err
|
||||
}
|
||||
apiKey := src.APIKeyRef
|
||||
if s.secrets != nil && apiKey != "" {
|
||||
if resolved, rerr := s.secrets.Get(ctx, apiKey); rerr == nil && resolved != "" {
|
||||
apiKey = resolved
|
||||
}
|
||||
}
|
||||
arrRoots, err := s.rootFolders.RootFolders(ctx, src.BaseURL, apiKey)
|
||||
if err != nil {
|
||||
return RewriteSuggestions{}, fmt.Errorf("autoscan: list arr root folders: %w", err)
|
||||
}
|
||||
siloPaths, err := s.folders.ListFolderPaths(ctx)
|
||||
if err != nil {
|
||||
return RewriteSuggestions{}, fmt.Errorf("autoscan: list silo folders: %w", err)
|
||||
}
|
||||
return suggestRewrites(arrRoots, siloPaths, src.PathRewrites), nil
|
||||
}
|
||||
```
|
||||
Add `GetSource` to `fakeStore` in `service_test.go` (`func (f *fakeStore) GetSource(context.Context, string) (*Source, error) { return nil, nil }`).
|
||||
|
||||
- [ ] **Step 4.4: Run tests — expect pass; commit**
|
||||
|
||||
```bash
|
||||
go test ./internal/autoscan/...
|
||||
gofmt -l internal/autoscan/service.go internal/autoscan/service_test.go
|
||||
git add internal/autoscan/service.go internal/autoscan/service_test.go
|
||||
git commit -m "feat(autoscan): Service.SuggestRewrites"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — API endpoint
|
||||
|
||||
### Task 5: Handler + route
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/api/handlers/autoscan.go`
|
||||
- Modify: `internal/api/router.go`
|
||||
- Modify: `internal/api/handlers/autoscan_test.go`
|
||||
|
||||
- [ ] **Step 5.1: Extend the handler's service interface + add the handler method**
|
||||
|
||||
In `autoscan.go`, add `SuggestRewrites` to the `autoscanTriggerer` interface:
|
||||
```go
|
||||
type autoscanTriggerer interface {
|
||||
PollOnce(ctx context.Context) error
|
||||
SuggestRewrites(ctx context.Context, integrationID string) (autoscan.RewriteSuggestions, error)
|
||||
}
|
||||
```
|
||||
Add the handler:
|
||||
```go
|
||||
func (h *AutoscanHandler) HandleRewriteSuggestions(w http.ResponseWriter, r *http.Request) {
|
||||
id := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
suggestions, err := h.svc.SuggestRewrites(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, autoscan.ErrIntegrationNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Autoscan source not found")
|
||||
return
|
||||
}
|
||||
// arr unreachable / bad key / folder list failure
|
||||
writeError(w, http.StatusBadGateway, "arr_unreachable", "Could not load root folders from the arr instance")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, suggestions)
|
||||
}
|
||||
```
|
||||
(`errors`, `autoscan`, `chi`, `strings`, `writeError`, `writeJSON` already imported.)
|
||||
|
||||
- [ ] **Step 5.2: Mount the route** in `router.go`, inside the autoscan route block:
|
||||
```go
|
||||
r.Get("/autoscan/sources/{id}/rewrite-suggestions", autoscanHandler.HandleRewriteSuggestions)
|
||||
```
|
||||
|
||||
- [ ] **Step 5.3: Wire the service deps** in `router.go` where `autoscanSvc` is built (after `autoscan.NewService(...)`):
|
||||
```go
|
||||
autoscanSvc.SetRewriteResolvers(
|
||||
autoscan.NewArrRootFolderClient(nil),
|
||||
autoscan.NewCatalogFolderLister(deps.FolderRepo),
|
||||
)
|
||||
```
|
||||
(This is inside the existing `if deps.FolderRepo != nil && deps.LibraryScanQueue != nil` block, so `deps.FolderRepo` is non-nil here.)
|
||||
|
||||
- [ ] **Step 5.4: Update the handler test fake**
|
||||
|
||||
In `autoscan_test.go`, the fake satisfying `autoscanTriggerer` (e.g. `fakeAutoscanTriggerer`) gains:
|
||||
```go
|
||||
func (f *fakeAutoscanTriggerer) SuggestRewrites(context.Context, string) (autoscan.RewriteSuggestions, error) {
|
||||
return autoscan.RewriteSuggestions{}, nil
|
||||
}
|
||||
```
|
||||
Add a test asserting `HandleRewriteSuggestions` returns 200 with JSON for a known id (mirror the existing handler tests).
|
||||
|
||||
- [ ] **Step 5.5: Verify (limited) + commit**
|
||||
|
||||
```bash
|
||||
# ensure the Go toolchain is on PATH
|
||||
go vet ./internal/autoscan/...
|
||||
gofmt -l internal/api/handlers/autoscan.go internal/api/router.go internal/api/handlers/autoscan_test.go
|
||||
# internal/api cannot compile here (libvips) — Docker verifies later.
|
||||
git add internal/api/handlers/autoscan.go internal/api/router.go internal/api/handlers/autoscan_test.go
|
||||
git commit -m "feat(autoscan): rewrite-suggestions endpoint"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Frontend
|
||||
|
||||
### Task 6: Types + hook
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/api/types.ts`
|
||||
- Modify: `web/src/hooks/queries/useAutoscan.ts`
|
||||
|
||||
- [ ] **Step 6.1: Types** — add to `types.ts`:
|
||||
```ts
|
||||
export interface AutoscanProposedRewrite {
|
||||
from: string;
|
||||
to: string;
|
||||
match_depth: number;
|
||||
}
|
||||
export interface AutoscanAmbiguousRoot {
|
||||
root: string;
|
||||
candidates: string[];
|
||||
}
|
||||
export interface AutoscanRewriteSuggestions {
|
||||
proposed: AutoscanProposedRewrite[];
|
||||
unmatched: string[];
|
||||
ambiguous: AutoscanAmbiguousRoot[];
|
||||
covered: string[];
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6.2: Hook** — add to `useAutoscan.ts` a mutation that fetches suggestions for an id:
|
||||
```ts
|
||||
export function useAutoscanRewriteSuggestions() {
|
||||
return useMutation({
|
||||
mutationFn: (id: string) =>
|
||||
api<AutoscanRewriteSuggestions>(`/admin/autoscan/sources/${id}/rewrite-suggestions`),
|
||||
onError: (err) =>
|
||||
toast.error(err instanceof Error ? err.message : "Could not sync rewrites from the arr instance"),
|
||||
});
|
||||
}
|
||||
```
|
||||
Match the import/`api`/`toast`/`useMutation` conventions already in the file (it imports `api` and `toast` already; reuse them; import the new type).
|
||||
|
||||
- [ ] **Step 6.3: Lint + commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b && pnpm run lint && pnpm exec prettier --check src/api/types.ts src/hooks/queries/useAutoscan.ts
|
||||
git add web/src/api/types.ts web/src/hooks/queries/useAutoscan.ts
|
||||
git commit -m "feat(web): autoscan rewrite-suggestions types and hook"
|
||||
```
|
||||
|
||||
### Task 7: Sync button + preview in AutoscanSourceEditor
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/pages/AdminRequests.tsx`
|
||||
|
||||
- [ ] **Step 7.1: Add the button + preview**
|
||||
|
||||
In `AutoscanSourceEditor`:
|
||||
- Call `const suggest = useAutoscanRewriteSuggestions();` and add local state `const [preview, setPreview] = useState<AutoscanRewriteSuggestions | null>(null);` and `const [selected, setSelected] = useState<Set<string>>(new Set());`.
|
||||
- Add a **"Sync from arr"** `Button` beside Save: `onClick={async () => { const s = await suggest.mutateAsync(source.integration_id); setPreview(s); setSelected(new Set(s.proposed.map((p) => p.from))); }}` (disabled while `suggest.isPending`).
|
||||
- When `preview` is set, render a panel (reuse the file's `Dialog` or an inline bordered panel) with three sections:
|
||||
- **Proposed:** each `proposed[]` row — a checkbox bound to `selected.has(p.from)` (toggle updates the set), `from → to`, and a `Badge` showing confidence: `match_depth >= 2 ? "${match_depth} segments" : "1 segment — weak"` (destructive/secondary variant for weak).
|
||||
- **Unmatched:** muted list of `unmatched[]` ("no Silo match — add manually if needed").
|
||||
- **Ambiguous:** list of `ambiguous[]` (`root` + its `candidates` joined) — info only.
|
||||
- An **"Add selected to rewrites"** button that merges the checked proposals into the editor's existing rewrites state, deduped by `from` (skip any whose `from` already exists), then `setPreview(null)`. Do NOT auto-save — the admin then clicks the existing **Save**.
|
||||
- Keep the existing manual path-rewrite editor unchanged; sync only appends rows to it.
|
||||
|
||||
Reuse existing `Button`/`Badge`/`Field`/`Dialog` primitives and the editor's existing rewrites-state setter.
|
||||
|
||||
- [ ] **Step 7.2: Lint + commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b && pnpm run lint && pnpm exec prettier --check src/pages/AdminRequests.tsx
|
||||
git add web/src/pages/AdminRequests.tsx
|
||||
git commit -m "feat(web): autoscan sync-rewrites preview"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 — Verification
|
||||
|
||||
### Task 8: Full verification
|
||||
|
||||
- [ ] **Step 8.1: Go**
|
||||
|
||||
```bash
|
||||
# ensure the Go toolchain is on PATH
|
||||
go test ./internal/autoscan/... 2>&1 | tail
|
||||
go vet ./internal/autoscan/...
|
||||
gofmt -l internal/autoscan internal/api/handlers/autoscan.go internal/api/router.go
|
||||
```
|
||||
Expected: all pass; vet/gofmt clean.
|
||||
|
||||
- [ ] **Step 8.2: Frontend**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b && pnpm run lint && pnpm run format:check
|
||||
```
|
||||
|
||||
- [ ] **Step 8.3: Docker build (closes the libvips gap)**
|
||||
|
||||
```bash
|
||||
docker build --build-arg BUILD_REVISION=$(git rev-parse HEAD) --build-arg BUILD_DIRTY=false -t silo-server:autoscan .
|
||||
```
|
||||
Expected: exit 0 (handlers + router + frontend compile).
|
||||
|
||||
- [ ] **Step 8.4: Smoke (optional)**
|
||||
|
||||
On a deploy with arr instances configured: open an instance in the Autoscan tab, click **Sync from arr**, confirm the preview shows proposed (with confidence) + unmatched + ambiguous, "Add selected" appends deduped rows, and Save persists them.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes (resolved)
|
||||
|
||||
- **Spec §1 matcher** → Task 1 (incl. normalization, covered-first, suffix depth, unlike-deployment fixtures). **§2 endpoint/service** → Tasks 2–5 (`GetSource`, `RootFolderClient`/`FolderLister`, `SuggestRewrites`, endpoint, 404/502 mapping). **§3 frontend** → Tasks 6–7. **§4 testing** → Tasks 1,4,5 + Task 8.
|
||||
- **Manual-preserving**: sync only *appends* deduped rows to the editor; the admin saves via the existing `PUT` (Task 7). Never auto-writes.
|
||||
- **Generic**: `suggestRewrites` takes plain string slices, no deployment constants; tests include `/data/Movies` and `/srv/tv` fixtures (Task 1).
|
||||
- **Optional deps**: `SetRewriteResolvers` keeps `main.go`'s poll service untouched; `SuggestRewrites` errors if unset (Task 4).
|
||||
- **Type consistency**: `RewriteSuggestions`/`ProposedRewrite`/`AmbiguousRoot` (Go) ↔ `AutoscanRewriteSuggestions`/`AutoscanProposedRewrite`/`AutoscanAmbiguousRoot` (TS); `match_depth` JSON tag matches.
|
||||
@@ -0,0 +1,375 @@
|
||||
# `scan_source.v1` SDK Capability — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a new, additive `scan_source.v1` plugin capability to `silo-plugin-sdk` so the Silo host can pull changed paths from a scan-source plugin on a timer.
|
||||
|
||||
**Architecture:** Mirror the existing `scheduled_task.v1` capability exactly — a single-RPC, request/response gRPC service defined in a `.proto`, generated into `pkg/pluginproto`, then registered in three Go helper sites (`capability.KnownTypes`, the `runtime.CapabilityServers` struct, and the `runtime.GRPCPlugin` server registration). Nothing existing is modified in shape; only new symbols are added. Tag a new SDK release so the host and the arr plugin can build against it.
|
||||
|
||||
**Tech Stack:** Go, Protocol Buffers (proto3), `buf` codegen (`buf.gen.yaml`), `google.golang.org/protobuf`, `google.golang.org/grpc`, the `hashicorp/go-plugin` runtime the SDK wraps.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites (read before starting)
|
||||
|
||||
This plan targets the **`silo-plugin-sdk`** repository (Go module `github.com/Silo-Server/silo-plugin-sdk`), which is **not currently checked out** in this workspace — only `silo-server` and `silo-plugin-tmdb` are. Before executing:
|
||||
|
||||
1. Clone `silo-plugin-sdk` as a writable working copy beside the other repos. Commands below assume **the silo-plugin-sdk repository root is the cwd**.
|
||||
2. Ensure the proto toolchain is available: the `make proto` target auto-installs `buf`, `protoc-gen-go@v1.36.11`, and `protoc-gen-go-grpc@v1.6.1` into `$(GO_BIN)`, but it requires the **`protoc`** binary to already be on `PATH` (it errors `protoc is required` otherwise). Install `protoc` if absent.
|
||||
3. This SDK is the **long pole**: the silo-server host plan and the arr-plugin plan both depend on the version tagged at the end of this plan (Task 6). Do not start those until this is tagged.
|
||||
|
||||
**Pattern reference (current `scheduled_task.v1`, the template this mirrors):**
|
||||
- Proto: `proto/silo/plugin/v1/scheduled_task.proto`
|
||||
- Generated: `pkg/pluginproto/silo/plugin/v1/scheduled_task.pb.go` + `scheduled_task_grpc.pb.go`
|
||||
- Capability const + allowlist: `pkg/pluginsdk/capability/capability.go`
|
||||
- Runtime wiring: `pkg/pluginsdk/runtime/runtime.go` (struct field, `Client` accessor, `GRPCServer` registration)
|
||||
- Manifest acceptance test pattern: `pkg/pluginsdk/manifest/manifest_test.go`
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `proto/silo/plugin/v1/scan_source.proto` | The `ScanSource` service + `PollChanges` request/response messages | Create |
|
||||
| `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go` | Generated message types | Create (codegen) |
|
||||
| `pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go` | Generated client/server stubs | Create (codegen) |
|
||||
| `pkg/pluginsdk/capability/capability.go` | `ScanSource` const + add to `KnownTypes` allowlist | Modify |
|
||||
| `pkg/pluginsdk/capability/capability_test.go` | Assert `ScanSource` is a known type | Create or Modify |
|
||||
| `pkg/pluginsdk/runtime/runtime.go` | `CapabilityServers.ScanSource` field, `Client.ScanSource()` accessor, `GRPCServer` registration | Modify |
|
||||
| `pkg/pluginsdk/runtime/runtime_test.go` | Assert a `ScanSource` server registers without error | Create or Modify |
|
||||
| `pkg/pluginsdk/manifest/manifest_test.go` | Assert a manifest declaring `scan_source.v1` loads | Modify |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Define and generate the `scan_source.v1` proto
|
||||
|
||||
**Files:**
|
||||
- Create: `proto/silo/plugin/v1/scan_source.proto`
|
||||
- Create (via codegen): `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go`, `pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go`
|
||||
|
||||
- [ ] **Step 1: Write the proto**
|
||||
|
||||
Create `proto/silo/plugin/v1/scan_source.proto` (the `go_package` option must match the sibling protos exactly so generated code lands in package `pluginv1`):
|
||||
|
||||
```proto
|
||||
syntax = "proto3";
|
||||
|
||||
package silo.plugin.v1;
|
||||
|
||||
option go_package = "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1;pluginv1";
|
||||
|
||||
// ScanSource lets the host pull changed filesystem paths from a provider
|
||||
// (e.g. Sonarr/Radarr, inotify, Ceph). Pull only: the host owns the timer and
|
||||
// calls PollChanges; the plugin never initiates.
|
||||
service ScanSource {
|
||||
rpc PollChanges(PollChangesRequest) returns (PollChangesResponse);
|
||||
}
|
||||
|
||||
message PollChangesRequest {
|
||||
// Which configured scan_source capability instance is being polled.
|
||||
string capability_id = 1;
|
||||
// Opaque continuation token from the previous PollChanges. Empty on first
|
||||
// run, which the provider treats as "start from now" (do not replay history).
|
||||
string marker = 2;
|
||||
}
|
||||
|
||||
message PollChangesResponse {
|
||||
// Absolute paths already translated into Silo's filesystem namespace (the
|
||||
// plugin has applied its own path rewrites). Files or directories.
|
||||
repeated string changed_paths = 1;
|
||||
// Opaque continuation token. The host stores it verbatim and echoes it back
|
||||
// on the next PollChanges; the host never parses it.
|
||||
string next_marker = 2;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Generate the Go code**
|
||||
|
||||
Run: `make proto`
|
||||
Expected: exits 0; creates `pkg/pluginproto/silo/plugin/v1/scan_source.pb.go` and `scan_source_grpc.pb.go`. (If it prints `protoc is required`, install `protoc` and re-run.)
|
||||
|
||||
- [ ] **Step 3: Verify the generated symbols exist and compile**
|
||||
|
||||
Run: `go build ./... && grep -l "func NewScanSourceClient" pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go && grep -l "RegisterScanSourceServer" pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go`
|
||||
Expected: build succeeds; both `grep`s print the filename (confirming `ScanSourceClient`, `ScanSourceServer`, `NewScanSourceClient`, `RegisterScanSourceServer` were generated — `require_unimplemented_servers=false` in `buf.gen.yaml` means no forced `UnimplementedScanSourceServer` embedding).
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add proto/silo/plugin/v1/scan_source.proto pkg/pluginproto/silo/plugin/v1/scan_source.pb.go pkg/pluginproto/silo/plugin/v1/scan_source_grpc.pb.go
|
||||
git commit -m "feat(proto): add scan_source.v1 capability service"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Register the capability type in the allowlist
|
||||
|
||||
The manifest loader rejects capability types not in `capability.KnownTypes` (see `TestLoadRejectsUnknownCapabilityType`). A manifest declaring `scan_source.v1` cannot load until the type is registered here.
|
||||
|
||||
**Files:**
|
||||
- Modify: `pkg/pluginsdk/capability/capability.go`
|
||||
- Create or Modify: `pkg/pluginsdk/capability/capability_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Append to `pkg/pluginsdk/capability/capability_test.go` (create the file with this `package capability` header if it does not exist):
|
||||
|
||||
```go
|
||||
package capability
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestScanSourceIsKnownType(t *testing.T) {
|
||||
if ScanSource != "scan_source.v1" {
|
||||
t.Fatalf("ScanSource const = %q, want %q", ScanSource, "scan_source.v1")
|
||||
}
|
||||
found := false
|
||||
for _, k := range KnownTypes {
|
||||
if k == ScanSource {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("ScanSource (%q) missing from KnownTypes %v", ScanSource, KnownTypes)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/capability/ -run TestScanSourceIsKnownType -v`
|
||||
Expected: FAIL — `undefined: ScanSource` (the const does not exist yet).
|
||||
|
||||
- [ ] **Step 3: Add the const and allowlist entry**
|
||||
|
||||
In `pkg/pluginsdk/capability/capability.go`, add the const inside the existing `const (...)` block (after `EbookBackend`):
|
||||
|
||||
```go
|
||||
EbookBackend = "ebook_backend.v1"
|
||||
ScanSource = "scan_source.v1"
|
||||
```
|
||||
|
||||
and add it to the `KnownTypes` slice (append `ScanSource` to the existing list):
|
||||
|
||||
```go
|
||||
ScanSource,
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/capability/ -run TestScanSourceIsKnownType -v`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add pkg/pluginsdk/capability/capability.go pkg/pluginsdk/capability/capability_test.go
|
||||
git commit -m "feat(capability): register scan_source.v1 as a known capability type"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Wire the capability into the runtime
|
||||
|
||||
Add the `ScanSource` server to `CapabilityServers`, a `Client.ScanSource()` accessor, and the conditional registration in `GRPCServer` — mirroring `ScheduledTask` in `pkg/pluginsdk/runtime/runtime.go`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `pkg/pluginsdk/runtime/runtime.go`
|
||||
- Create or Modify: `pkg/pluginsdk/runtime/runtime_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Append to `pkg/pluginsdk/runtime/runtime_test.go` (create with the `package runtime` header if absent). This test constructs a real `*grpc.Server`, supplies a minimal `ScanSource` server, and asserts `GRPCServer` registers it without error:
|
||||
|
||||
```go
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type stubScanSource struct{}
|
||||
|
||||
func (stubScanSource) PollChanges(context.Context, *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
return &pluginv1.PollChangesResponse{}, nil
|
||||
}
|
||||
|
||||
func TestGRPCServerRegistersScanSource(t *testing.T) {
|
||||
p := &GRPCPlugin{Servers: CapabilityServers{
|
||||
Runtime: stubRuntime{}, // existing stub used by sibling runtime tests
|
||||
ScanSource: stubScanSource{},
|
||||
}}
|
||||
srv := grpc.NewServer()
|
||||
if err := p.GRPCServer(nil, srv); err != nil {
|
||||
t.Fatalf("GRPCServer with ScanSource = %v, want nil", err)
|
||||
}
|
||||
if _, ok := srv.GetServiceInfo()["silo.plugin.v1.ScanSource"]; !ok {
|
||||
t.Fatalf("ScanSource service not registered; got %v", srv.GetServiceInfo())
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note: `GRPCServer` requires `Servers.Runtime` to be non-nil (it errors otherwise). Reuse the existing runtime stub the sibling tests use. If no such stub exists in `runtime_test.go`, add a minimal one:
|
||||
|
||||
```go
|
||||
type stubRuntime struct{ pluginv1.RuntimeServer }
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/runtime/ -run TestGRPCServerRegistersScanSource -v`
|
||||
Expected: FAIL — `unknown field 'ScanSource' in struct literal of type CapabilityServers` (the field does not exist yet).
|
||||
|
||||
- [ ] **Step 3: Add the struct field**
|
||||
|
||||
In `pkg/pluginsdk/runtime/runtime.go`, add to the `CapabilityServers` struct (after the `ScheduledTask` field):
|
||||
|
||||
```go
|
||||
ScheduledTask pluginv1.ScheduledTaskServer
|
||||
ScanSource pluginv1.ScanSourceServer
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add the `Client` accessor**
|
||||
|
||||
After the existing `func (c *Client) ScheduledTask() ...` method, add:
|
||||
|
||||
```go
|
||||
func (c *Client) ScanSource() pluginv1.ScanSourceClient {
|
||||
return pluginv1.NewScanSourceClient(c.conn)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Add the conditional registration in `GRPCServer`**
|
||||
|
||||
In `func (p *GRPCPlugin) GRPCServer(...)`, after the `ScheduledTask` registration block, add:
|
||||
|
||||
```go
|
||||
if p.Servers.ScheduledTask != nil {
|
||||
pluginv1.RegisterScheduledTaskServer(server, p.Servers.ScheduledTask)
|
||||
}
|
||||
if p.Servers.ScanSource != nil {
|
||||
pluginv1.RegisterScanSourceServer(server, p.Servers.ScanSource)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run test to verify it passes**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/runtime/ -run TestGRPCServerRegistersScanSource -v`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add pkg/pluginsdk/runtime/runtime.go pkg/pluginsdk/runtime/runtime_test.go
|
||||
git commit -m "feat(runtime): serve and dial scan_source.v1 capability"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Prove a `scan_source.v1` manifest loads
|
||||
|
||||
Guards the end-to-end path: a plugin manifest declaring the new capability must pass the loader (it would not before Task 2's allowlist entry).
|
||||
|
||||
**Files:**
|
||||
- Modify: `pkg/pluginsdk/manifest/manifest_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
The test file is package `manifest_test` (external) and loads from a JSON literal via `manifest.Load([]byte)`. Append this test, mirroring `TestLoadAcceptsRequestRouterCapability` exactly with the type swapped to `scan_source.v1`:
|
||||
|
||||
```go
|
||||
func TestLoadAcceptsScanSourceCapability(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"plugin_id": "silo.example",
|
||||
"version": "1.0.0",
|
||||
"silo_api_version": "v1",
|
||||
"capabilities": [
|
||||
{"type": "scan_source.v1", "id": "arr", "display_name": "X", "description": "Y"}
|
||||
]
|
||||
}`)
|
||||
m, err := manifest.Load(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("Load returned unexpected error: %v", err)
|
||||
}
|
||||
if len(m.GetCapabilities()) != 1 {
|
||||
t.Fatalf("expected 1 capability, got %d", len(m.GetCapabilities()))
|
||||
}
|
||||
if got := m.GetCapabilities()[0].GetType(); got != "scan_source.v1" {
|
||||
t.Fatalf("capability type = %q, want scan_source.v1", got)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails (before this task's dependency was in place) / passes now**
|
||||
|
||||
Run: `go test ./pkg/pluginsdk/manifest/ -run TestLoadAcceptsScanSourceCapability -v`
|
||||
Expected: PASS (Task 2 already added `scan_source.v1` to `KnownTypes`). If it FAILS with an unknown-capability-type error, Task 2 was not completed correctly — fix that first.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add pkg/pluginsdk/manifest/manifest_test.go
|
||||
git commit -m "test(manifest): accept scan_source.v1 capability in manifests"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Full build + test sweep
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Build everything**
|
||||
|
||||
Run: `go build ./...`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 2: Run the full SDK test suite**
|
||||
|
||||
Run: `go test ./...`
|
||||
Expected: all packages `ok`. Confirms the additive change broke no existing capability, manifest, or runtime test.
|
||||
|
||||
- [ ] **Step 3: Vet + format**
|
||||
|
||||
Run: `go vet ./... && gofmt -l pkg proto 2>/dev/null; test -z "$(gofmt -l pkg)"`
|
||||
Expected: vet clean; `gofmt -l pkg` prints nothing.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Tag the release
|
||||
|
||||
Downstream repos (silo-server host, arr plugin) consume the SDK as a tagged module version.
|
||||
|
||||
**Files:** none (release action)
|
||||
|
||||
- [ ] **Step 1: Confirm the version to tag**
|
||||
|
||||
Check the latest tag: `git tag --list 'v*' | sort -V | tail -3`. The current consumed version is `v0.4.0`; this additive feature warrants a minor bump to **`v0.5.0`** (no breaking changes).
|
||||
|
||||
- [ ] **Step 2: Tag and push**
|
||||
|
||||
```bash
|
||||
git tag v0.5.0
|
||||
git push origin v0.5.0
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Record the contract for downstream plans**
|
||||
|
||||
Note in the host and arr-plugin plans that they must require `github.com/Silo-Server/silo-plugin-sdk v0.5.0` and use:
|
||||
- `capability.ScanSource` (`"scan_source.v1"`)
|
||||
- `runtime.CapabilityServers{ ScanSource: <impl> }` (plugin side)
|
||||
- `client.ScanSource()` → `pluginv1.ScanSourceClient.PollChanges(ctx, &pluginv1.PollChangesRequest{CapabilityId, Marker})` (host side)
|
||||
- response fields `PollChangesResponse.ChangedPaths []string`, `PollChangesResponse.NextMarker string`
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** Implements spec §5 (the `scan_source.v1` contract: `PollChanges`, opaque marker, Silo-native paths) and §11 (additive-only; registered in `KnownTypes`; no existing capability modified). Spec §12 build-order step 1 (tag the SDK) is Task 6.
|
||||
- **Out of scope here:** the host engine (spec §7), connections (§8), the Autoscan category (§9), and the arr plugin (§10) are separate per-repo plans built against the `v0.5.0` tag produced by Task 6.
|
||||
- **Marker is opaque end-to-end:** the proto carries `marker`/`next_marker` as plain strings with no server-side interpretation, satisfying the "any provider's bookmark" requirement.
|
||||
@@ -0,0 +1,265 @@
|
||||
# Sonarr/Radarr Scan-Source Plugin Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build a new installable Silo plugin that implements `scan_source.v1` for Sonarr/Radarr: when the host pulls, it polls arr `/history`, extracts imported + renamed paths, applies its own path rewrites, and returns Silo-native paths + a marker.
|
||||
|
||||
**Architecture:** A standalone Go module structured like `silo-plugin-tmdb` — a `manifest.json` declaring `scan_source.v1`, a `main.go` serving `runtime.CapabilityServers{Runtime, ScanSource}` via `runtime.Serve`, and the arr-specific logic ported from the closed silo-server PR #43 (`internal/autoscan/history.go`, `rewrite.go`, `suggest.go`).
|
||||
|
||||
**Tech Stack:** Go, `github.com/Silo-Server/silo-plugin-sdk` (`v0.5.0`, the `scan_source.v1` capability), `hashicorp/go-plugin` runtime (via the SDK), arr HTTP API (`X-Api-Key`).
|
||||
|
||||
Commands assume the **new plugin repository root** is the cwd (the `silo-plugin-autoscan-arr` checkout, a sibling of this repo like the `silo-plugin-sdk` checkout).
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **SDK `v0.5.0`.** Needs `scan_source.v1` from `silo-plugin-sdk` PR #2. Until tagged, depend on the local checkout via replace:
|
||||
```bash
|
||||
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk
|
||||
```
|
||||
This points at the `silo-plugin-sdk` checkout (a sibling of this repo). Task 8 finalizes to `v0.5.0`.
|
||||
2. **Template:** the `silo-plugin-tmdb` checkout (a sibling of this repo) — copy its `main.go` runtime scaffold, `Makefile`, and `manifest.json` shape.
|
||||
3. **Salvage source:** the arr logic is on silo-server `main` at `internal/autoscan/history.go` (imports + renames extraction, bounded window), `rewrite.go` (`applyRewrites`, `normalizeSeparators`), `suggest.go`/`suggest_deps.go` (root-folder suffix-match suggester). Port these verbatim where possible.
|
||||
4. **New repo:** create `Silo-Server/silo-plugin-autoscan-arr` (or fork pattern). The plugin is registered in the `silo-plugins` catalog in a separate follow-up (Task 9 notes it).
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `go.mod` | module `github.com/Silo-Server/silo-plugin-autoscan-arr`; SDK dep | Create |
|
||||
| `manifest.json` | declares `scan_source.v1` capability + `global_config_schema` (path rewrites only; connection arrives per request) | Create |
|
||||
| `main.go` | `runtimeServer` (GetManifest/Configure) + `scanSourceServer` (PollChanges); `runtime.Serve` | Create |
|
||||
| `internal/arr/history.go` | poll `/history`, imports + renames, bounded window | Create (port) |
|
||||
| `internal/arr/rewrite.go` | `applyRewrites`, `normalizeSeparators` | Create (port) |
|
||||
| `internal/arr/client.go` | arr HTTP client (`X-Api-Key`, response cap, timeout) | Create |
|
||||
| `internal/config/config.go` | parse Configure payload → `{rewrites}` (connection comes per request, not config) | Create |
|
||||
| `Makefile` | build (cross-platform), like tmdb | Create |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Scaffold the plugin (serves an empty `scan_source.v1`)
|
||||
|
||||
**Files:** Create `go.mod`, `manifest.json`, `main.go`, `Makefile`.
|
||||
|
||||
- [ ] **Step 1: Module + SDK dep**
|
||||
|
||||
```bash
|
||||
go mod init github.com/Silo-Server/silo-plugin-autoscan-arr
|
||||
go mod edit -replace github.com/Silo-Server/silo-plugin-sdk=../silo-plugin-sdk # the silo-plugin-sdk checkout, a sibling of this repo
|
||||
go get github.com/Silo-Server/silo-plugin-sdk@v0.4.0 # replace points at the scan_source branch checkout
|
||||
```
|
||||
|
||||
- [ ] **Step 2: manifest.json**
|
||||
|
||||
Mirror tmdb's shape, declaring the scan source:
|
||||
|
||||
```json
|
||||
{
|
||||
"plugin_id": "silo.autoscan.arr",
|
||||
"version": "0.1.0",
|
||||
"checksum": "__CHECKSUM__",
|
||||
"silo_api_version": "v1",
|
||||
"supported_platforms": [
|
||||
{"os": "linux", "arch": "amd64"},
|
||||
{"os": "linux", "arch": "arm64"},
|
||||
{"os": "darwin", "arch": "arm64"}
|
||||
],
|
||||
"capabilities": [
|
||||
{
|
||||
"type": "scan_source.v1",
|
||||
"id": "arr",
|
||||
"display_name": "Sonarr / Radarr",
|
||||
"description": "Triggers Silo rescans from Sonarr/Radarr import and rename history."
|
||||
}
|
||||
],
|
||||
"global_config_schema": [
|
||||
{"key": "path_rewrites", "type": "json", "required": false, "display_name": "Path rewrites"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
> **Connection is NOT plugin config.** The arr base URL + API key are delivered by the host in each `PollChangesRequest.connection` (resolved from the operator's Autoscan connection — own creds or a reused Requests link). The plugin reads them from the request and stores no credentials. Only path rewrites (and other provider-specific behaviour) live in plugin config.
|
||||
|
||||
(Confirm the exact `global_config_schema` entry shape against `silo-plugin-sdk` `pkg/pluginsdk/config` — adapt field names to the SDK's config schema type.)
|
||||
|
||||
- [ ] **Step 3: main.go scaffold**
|
||||
|
||||
Copy tmdb's `runtimeServer` (GetManifest from embedded manifest; Configure stores config) and add a `scanSourceServer` returning empty for now:
|
||||
|
||||
```go
|
||||
type scanSourceServer struct{ cfg *config.Config }
|
||||
|
||||
func (s *scanSourceServer) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
return &pluginv1.PollChangesResponse{}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
manifest := mustLoadManifest()
|
||||
rt := &runtimeServer{manifest: manifest}
|
||||
runtime.Serve(runtime.ServeConfig{
|
||||
Servers: runtime.CapabilityServers{
|
||||
Runtime: rt,
|
||||
ScanSource: &scanSourceServer{cfg: rt.cfg},
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Build**
|
||||
|
||||
Run: `go build ./...`
|
||||
Expected: exits 0.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git init && git add -A
|
||||
git commit -m "feat: scaffold autoscan-arr scan_source.v1 plugin"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Config parsing (Configure → path rewrites only)
|
||||
|
||||
**Files:** Create `internal/config/config.go`, `internal/config/config_test.go`.
|
||||
|
||||
> Connection (base_url/api_key) is NOT config — it arrives per poll in `PollChangesRequest.connection` (Task 5). Plugin config carries only path rewrites.
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
|
||||
```go
|
||||
func TestParseConfig(t *testing.T) {
|
||||
cfg, err := Parse(map[string]string{
|
||||
"path_rewrites": `[{"from":"/mnt/arr/tv","to":"/mnt/media/tv"}]`,
|
||||
})
|
||||
if err != nil { t.Fatalf("Parse: %v", err) }
|
||||
if len(cfg.Rewrites) != 1 || cfg.Rewrites[0].From != "/mnt/arr/tv" { t.Fatalf("bad rewrites: %+v", cfg.Rewrites) }
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → fail.** `go test ./internal/config/` → undefined `Parse`.
|
||||
|
||||
- [ ] **Step 3: Implement** `Parse(map[string]string) (*Config, error)` with `Config{Rewrites []Rewrite}` where `Rewrite{From, To string}`; unmarshal `path_rewrites` JSON. Wire `runtimeServer.Configure` to store the parsed config so `scanSourceServer` reads the rewrites.
|
||||
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
|
||||
- [ ] **Step 5: Commit** `feat(config): parse arr connection and path rewrites`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: arr history client (imports + renames, bounded window)
|
||||
|
||||
**Files:** Create `internal/arr/client.go`, `internal/arr/history.go`, `internal/arr/history_test.go`.
|
||||
|
||||
Port from silo-server `main:internal/autoscan/history.go` (the `ChangedPaths` logic: `downloadFolderImported.importedPath`; `episodeFileRenamed`/`movieFileRenamed` → `path` + `sourcePath`; deletes ignored) and the client (`X-Api-Key`, 1 MiB cap, timeout). The marker is an RFC3339 timestamp; empty marker ⇒ "now".
|
||||
|
||||
- [ ] **Step 1: Failing test** — copy the salvaged `history_test.go` (`TestArrHistoryChangedPaths`: imports + both rename paths returned, `grabbed`/`episodeFileDeleted` ignored), renamed to this package, asserting the returned `(paths, nextMarker)`.
|
||||
|
||||
- [ ] **Step 2: Run → fail.**
|
||||
|
||||
- [ ] **Step 3: Implement** `ChangedPaths(ctx, baseURL, apiKey, since time.Time) (paths []string, newest time.Time, err error)` — port `history.go`, additionally returning the newest history timestamp as the next marker. Apply the bounded window (24h max-lookback floor + overlap buffer) here.
|
||||
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
|
||||
- [ ] **Step 5: Commit** `feat(arr): poll history for imports and renames`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Path rewrites (→ Silo-native paths)
|
||||
|
||||
**Files:** Create `internal/arr/rewrite.go`, `internal/arr/rewrite_test.go`.
|
||||
|
||||
Port `applyRewrites` (boundary-safe prefix: `path==trimmed || HasPrefix(path, trimmed+"/")`) and `normalizeSeparators` from silo-server `main:internal/autoscan/rewrite.go`.
|
||||
|
||||
- [ ] **Step 1: Failing test** — copy salvaged `rewrite_test.go` cases.
|
||||
- [ ] **Step 2: Run → fail.**
|
||||
- [ ] **Step 3: Implement** the port.
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
- [ ] **Step 5: Commit** `feat(arr): boundary-safe path rewrites`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Wire `PollChanges` end-to-end
|
||||
|
||||
**Files:** Modify `main.go` (`scanSourceServer.PollChanges`), add `main_test.go`.
|
||||
|
||||
- [ ] **Step 1: Failing test** — a `scanSourceServer` (rewrites configured) called with a `PollChangesRequest` whose `Connection` points at a stub arr server (httptest); assert `PollChanges` with empty marker returns rewritten Silo-native paths and a non-empty `next_marker`, and that the request actually hit the stub at `connection.base_url` with `X-Api-Key: connection.api_key`. A second call with the returned marker queries arr with the right `since`. Also assert a request with a nil/empty `Connection` returns an error (no credentials to poll with).
|
||||
|
||||
- [ ] **Step 2: Run → fail.**
|
||||
|
||||
- [ ] **Step 3: Implement** — read the connection from the request, not from config:
|
||||
|
||||
```go
|
||||
func (s *scanSourceServer) PollChanges(ctx context.Context, req *pluginv1.PollChangesRequest) (*pluginv1.PollChangesResponse, error) {
|
||||
conn := req.GetConnection()
|
||||
if conn == nil || conn.GetBaseUrl() == "" {
|
||||
return nil, fmt.Errorf("scan_source: no connection supplied")
|
||||
}
|
||||
since := time.Time{} // empty marker => now (history client floors to now)
|
||||
if m := req.GetMarker(); m != "" {
|
||||
if t, err := time.Parse(time.RFC3339, m); err == nil { since = t }
|
||||
}
|
||||
raw, newest, err := arr.ChangedPaths(ctx, conn.GetBaseUrl(), conn.GetApiKey(), since)
|
||||
if err != nil { return nil, err }
|
||||
out := make([]string, 0, len(raw))
|
||||
for _, p := range raw {
|
||||
out = append(out, arr.ApplyRewrites(arr.NormalizeSeparators(p), s.cfg.Rewrites))
|
||||
}
|
||||
return &pluginv1.PollChangesResponse{ChangedPaths: out, NextMarker: newest.UTC().Format(time.RFC3339)}, nil
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run → pass.**
|
||||
|
||||
- [ ] **Step 5: Commit** `feat: PollChanges returns Silo-native paths from arr history`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Rewrite suggester ("Sync from arr") — root folders
|
||||
|
||||
**Files:** Create `internal/arr/suggest.go`, `suggest_test.go`.
|
||||
|
||||
Port `suggestRewrites` (suffix-match arr root folders → Silo folders) from `main:internal/autoscan/suggest.go`. **Open question (spec §14 risk #2):** how the operator triggers "Sync" and how the plugin reaches Silo's media-folder list.
|
||||
|
||||
- [ ] **Step 1:** Decide the delivery mechanism — recommended: the plugin reads Silo's folder list via the host library-listing service (`pluginhost` library lister, reachable through the SDK `runtime.Host()` client). Confirm that path exists in `v0.5.0`; if not, scope "Sync" to a follow-up and ship Tasks 1–5 (manual rewrites only) first.
|
||||
- [ ] **Step 2:** Port the suffix-match suggester + its tests (unique/ambiguous/covered/no-op/normalization cases from `suggest_test.go`).
|
||||
- [ ] **Step 3:** Expose it — either via an `http_routes.v1` capability the admin UI calls, or as part of the plugin config flow. Document the chosen mechanism.
|
||||
- [ ] **Step 4:** Tests pass; commit `feat(arr): suffix-match rewrite suggester`.
|
||||
|
||||
> If §14 risk #2 is unresolved at execution time, **skip this task**, ship manual rewrites (Tasks 1–5), and file the suggester as a follow-up. The plugin is fully functional without it.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Cross-repo smoke test against the host
|
||||
|
||||
**Files:** none (verification).
|
||||
|
||||
- [ ] **Step 1:** Build the plugin binary; install it into a dev silo-server (with the Part-1 backend) via the plugin installer.
|
||||
- [ ] **Step 2:** Configure a source pointing at a real/stub arr; trigger an autoscan poll; confirm the host enqueues a scan for an imported path. (This exercises the full SDK contract end-to-end.)
|
||||
- [ ] **Step 3:** Note results; no commit unless fixes are needed.
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Finalize SDK dependency
|
||||
|
||||
- [ ] After `silo-plugin-sdk v0.5.0` is tagged: `go mod edit -dropreplace ...; go get .../silo-plugin-sdk@v0.5.0; go mod tidy; go build ./...`.
|
||||
- [ ] Commit `build: depend on silo-plugin-sdk v0.5.0`.
|
||||
- [ ] Do not publish/tag the plugin with a `replace` directive in `go.mod`.
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Catalog registration (follow-up, separate repo)
|
||||
|
||||
- [ ] Add `silo.autoscan.arr` to the `silo-plugins` catalog manifest so it is installable. This is a separate PR in the `silo-plugins` repo (not this module) — note it; out of scope for this plan's repo.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** §10 (arr plugin: history imports + renames, bounded window, rewrites, Silo-native paths) Tasks 3–5; §5 contract (`PollChanges`, opaque marker = RFC3339 timestamp, first-run "now") Task 5; "Sync from arr" §10/§14-risk-2 Task 6 (with an explicit skip path).
|
||||
- **Salvage:** history/rewrite/suggest ported from the closed PR (Tasks 3,4,6) — exactly the files the host backend plan deletes (Part-1 Task 8).
|
||||
- **Risk:** §14 risk #1 (one plugin installation per arr server vs. many) — this plugin holds one connection per installation/config; multiple arr servers ⇒ multiple installs unless the runtime supports multiple capability instances. §14 risk #2 (Sync UI) gated in Task 6 with a clean skip.
|
||||
- **Type consistency:** `Config{Rewrites}` (no creds — connection arrives in `PollChangesRequest.connection`), `Rewrite{From,To}`, `arr.ChangedPaths(ctx, baseURL, apiKey, since) ([]string, time.Time, error)`, `arr.ApplyRewrites`/`NormalizeSeparators`, `scanSourceServer.PollChanges` consistent across Tasks 2–6.
|
||||
@@ -0,0 +1,209 @@
|
||||
# Autoscan Admin UI Implementation Plan (Part 2 of 2)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Give Autoscan its own admin category (page + sidebar nav), separate from Requests: manage Connections (reuse a Requests arr server or enter own), Sources (per scan-source plugin: enable, interval, connection), and global settings — backed by the Part-1 v2 API.
|
||||
|
||||
**Architecture:** Extract the Autoscan tab out of `AdminRequests.tsx` into a standalone `AdminAutoscan.tsx` page; register a route and a sidebar item; rebuild `useAutoscan.ts` hooks + `types.ts` against the v2 endpoints (`/admin/autoscan/{settings,connections,sources,trigger,status}`).
|
||||
|
||||
**Tech Stack:** React + TypeScript, React Query (`@tanstack/react-query`), the project's `ui/` component library (`Tabs`, `Table`, `Card`, etc.), `react-router` routes in `App.tsx`, 2-space indent / double quotes / 100-col (`web/.prettierrc`).
|
||||
|
||||
Commands assume the silo-server repository root is the cwd. **Depends on Part 1 (backend)** being implemented (the v2 endpoints must exist).
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Part-1 backend merged/available** — the v2 endpoints and response shapes (connections with `request_integration_id`, sources with `marker`/`last_error`, settings with `default_poll_interval_seconds`).
|
||||
2. **Reference the existing autoscan tab** at `web/src/pages/AdminRequests.tsx` (the `autoscan` `TabsContent`, the `AutoscanTab`/`AutoscanSourceEditor` components) and hooks at `web/src/hooks/queries/useAutoscan.ts` — these are the starting material; they are reshaped, not discarded.
|
||||
3. **Run before MR:** `cd web && pnpm run lint && pnpm run format:check`. Note the 20 pre-existing failing frontend tests are unrelated (project memory); do not be alarmed if they appear — only newly-introduced failures matter.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Action |
|
||||
|---|---|---|
|
||||
| `web/src/api/types.ts` | `AutoscanConnection`, `AutoscanSource`, `AutoscanSettings` (v2 shapes) | Modify |
|
||||
| `web/src/hooks/queries/useAutoscan.ts` | hooks for settings/connections/sources/trigger/status | Rewrite |
|
||||
| `web/src/hooks/queries/keys.ts` | query keys for connections/sources | Modify |
|
||||
| `web/src/pages/AdminAutoscan.tsx` | the new standalone Autoscan page (Connections + Sources + Settings tabs) | Create |
|
||||
| `web/src/pages/admin/autoscan/ConnectionsPanel.tsx` | list/add/edit/delete connections (own or Requests-linked) | Create |
|
||||
| `web/src/pages/admin/autoscan/SourcesPanel.tsx` | per-source enable, interval, connection binding, status | Create |
|
||||
| `web/src/pages/AdminRequests.tsx` | remove the Autoscan tab | Modify |
|
||||
| `web/src/App.tsx` | add `<Route path="autoscan" element={<AdminAutoscan />} />` | Modify |
|
||||
| `web/src/components/AdminSidebar.tsx` | add the Autoscan nav item under "Content" | Modify |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: v2 types + hooks
|
||||
|
||||
**Files:** Modify `web/src/api/types.ts`, `web/src/hooks/queries/keys.ts`; rewrite `web/src/hooks/queries/useAutoscan.ts`.
|
||||
|
||||
- [ ] **Step 1: Types**
|
||||
|
||||
Add to `web/src/api/types.ts`:
|
||||
|
||||
```ts
|
||||
export interface AutoscanSettings {
|
||||
enabled: boolean;
|
||||
default_poll_interval_seconds: number;
|
||||
debounce_seconds: number;
|
||||
}
|
||||
|
||||
export interface AutoscanConnection {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
// present when the connection has its own credentials:
|
||||
base_url?: string;
|
||||
// present when linked to a Requests integration (live reuse):
|
||||
request_integration_id?: string | null;
|
||||
// NOTE: api_key_ref / resolved keys are never sent by the backend.
|
||||
}
|
||||
|
||||
export interface AutoscanSource {
|
||||
id: string;
|
||||
installation_id: number;
|
||||
capability_id: string;
|
||||
connection_id: string;
|
||||
enabled: boolean;
|
||||
poll_interval_seconds: number | null;
|
||||
last_run_at: string | null;
|
||||
last_error: string | null;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Hooks**
|
||||
|
||||
Rewrite `useAutoscan.ts` with: `useAutoscanSettings`/`useUpdateAutoscanSettings`, `useAutoscanConnections`/`useCreateAutoscanConnection`/`useUpdateAutoscanConnection`/`useDeleteAutoscanConnection`, `useAutoscanSources`/`useUpdateAutoscanSource`, `useAutoscanStatus`, `useTriggerAutoscan`. Follow the existing hook style (the current `useAutoscan.ts` is the template for query/mutation + invalidation). Add query keys in `keys.ts`.
|
||||
|
||||
- [ ] **Step 3: Typecheck**
|
||||
|
||||
Run: `cd web && pnpm exec tsc --noEmit`
|
||||
Expected: no type errors in the autoscan files.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add web/src/api/types.ts web/src/hooks/queries/useAutoscan.ts web/src/hooks/queries/keys.ts
|
||||
git commit -m "feat(web): autoscan v2 types and query hooks"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Connections panel
|
||||
|
||||
**Files:** Create `web/src/pages/admin/autoscan/ConnectionsPanel.tsx`.
|
||||
|
||||
- [ ] **Step 1: Build the panel**
|
||||
|
||||
A `Card` + `Table` listing connections (name, kind, and a "Reused from Requests" / "Own" badge derived from `request_integration_id`). An "Add connection" dialog offers two modes:
|
||||
- **Reuse from Requests** — a select populated from the existing request-integrations hook (filter to arr kinds), storing `request_integration_id`.
|
||||
- **Enter own** — name + URL + API key fields, posting own credentials.
|
||||
|
||||
Edit + delete actions per row. Never render any key material (the backend doesn't send it).
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck**
|
||||
|
||||
Run: `cd web && pnpm run lint && pnpm exec tsc --noEmit`
|
||||
Expected: clean.
|
||||
|
||||
- [ ] **Step 3: Commit** `feat(web): autoscan connections panel (reuse or own)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Sources panel
|
||||
|
||||
**Files:** Create `web/src/pages/admin/autoscan/SourcesPanel.tsx`.
|
||||
|
||||
- [ ] **Step 1: Build the panel**
|
||||
|
||||
A `Table` of sources, one row per installed `scan_source` capability instance: plugin/capability label, a connection `<select>` (bound to `connection_id`), an enable toggle, a poll-interval input (blank = settings default), and a status column (`last_run_at`, `last_error`). Reuse the spinner/"Syncing…" affordances from the old `AutoscanSourceEditor` where relevant. Wire to `useAutoscanSources`/`useUpdateAutoscanSource`.
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck** — clean.
|
||||
- [ ] **Step 3: Commit** `feat(web): autoscan sources panel`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: The Autoscan page + global settings + manual trigger
|
||||
|
||||
**Files:** Create `web/src/pages/AdminAutoscan.tsx`.
|
||||
|
||||
- [ ] **Step 1: Compose the page**
|
||||
|
||||
A page with a `Tabs` (or sections): **Sources**, **Connections**, **Settings**. Settings = global enable toggle, default poll interval, debounce. A "Run now" button calling `useTriggerAutoscan` (202 → toast "Autoscan triggered"). Mirror the header/layout of `AdminRequests.tsx` for visual consistency.
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck** — clean.
|
||||
- [ ] **Step 3: Commit** `feat(web): standalone Autoscan admin page`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Route + sidebar nav
|
||||
|
||||
**Files:** Modify `web/src/App.tsx`, `web/src/components/AdminSidebar.tsx`.
|
||||
|
||||
- [ ] **Step 1: Route**
|
||||
|
||||
In `web/src/App.tsx`, next to `<Route path="requests" element={<AdminRequests />} />` (l.374), add:
|
||||
|
||||
```tsx
|
||||
<Route path="autoscan" element={<AdminAutoscan />} />
|
||||
```
|
||||
|
||||
with the matching `import AdminAutoscan from "@/pages/AdminAutoscan";`.
|
||||
|
||||
- [ ] **Step 2: Sidebar item**
|
||||
|
||||
In `web/src/components/AdminSidebar.tsx`, under the **Content** group (after the Requests item, ~l.112), add:
|
||||
|
||||
```tsx
|
||||
{
|
||||
label: "Autoscan",
|
||||
icon: <RefreshCw className="h-[18px] w-[18px]" />,
|
||||
href: "/admin/autoscan",
|
||||
},
|
||||
```
|
||||
|
||||
(Import an appropriate icon from the icon set already used in the file.)
|
||||
|
||||
- [ ] **Step 3: Verify navigation**
|
||||
|
||||
Run: `cd web && pnpm run build`
|
||||
Expected: build succeeds. Manually confirm `/admin/autoscan` renders and the sidebar item highlights.
|
||||
|
||||
- [ ] **Step 4: Commit** `feat(web): route and sidebar nav for Autoscan category`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Remove the Autoscan tab from Requests
|
||||
|
||||
**Files:** Modify `web/src/pages/AdminRequests.tsx`.
|
||||
|
||||
- [ ] **Step 1: Excise the tab**
|
||||
|
||||
Remove the `<TabsTrigger value="autoscan">` (l.146) and its `<TabsContent value="autoscan">`, the `AutoscanTab`/`AutoscanSourceEditor` definitions (now superseded by the new panels), and `autoscan` from `ADMIN_REQUEST_TABS`. Remove now-unused imports.
|
||||
|
||||
- [ ] **Step 2: Lint/format/typecheck/build**
|
||||
|
||||
Run: `cd web && pnpm run lint && pnpm run format:check && pnpm run build`
|
||||
Expected: clean; build succeeds; no dangling references to removed components.
|
||||
|
||||
- [ ] **Step 3: Commit** `refactor(web): move Autoscan out of Requests into its own category`.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Final sweep
|
||||
|
||||
- [ ] **Step 1:** `cd web && pnpm run lint && pnpm run format:check && pnpm run build` — all clean.
|
||||
- [ ] **Step 2:** `cd web && pnpm test` — confirm no **newly** failing tests (the 20 pre-existing failures, per project memory, are unrelated; compare against baseline).
|
||||
- [ ] **Step 3:** Manual pass: add a connection (both modes), bind a source, toggle enable, "Run now", confirm status updates.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** §9 (Autoscan as its own category, decoupled from Requests; Connections reuse-or-own; per-source enable/interval/status) — Tasks 1–6. The reuse-from-Requests UI (Task 2) surfaces the soft link from spec §8.
|
||||
- **Salvage:** the old `AutoscanTab`/`AutoscanSourceEditor` and `useAutoscan.ts` are reshaped (Tasks 1–3), then the Requests tab is removed (Task 6).
|
||||
- **Out of scope:** plugin-side config UI (the path-rewrite table lives on the plugin's own settings screen per spec §10 — that is the arr-plugin plan / spec §14 risk #2, not this page).
|
||||
- **Type consistency:** `AutoscanConnection{request_integration_id?}`, `AutoscanSource{connection_id,poll_interval_seconds,last_error}`, `AutoscanSettings{default_poll_interval_seconds}` used consistently across hooks (Task 1) and panels (Tasks 2–4).
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,174 @@
|
||||
# Audiobooks Stacked PR Split Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace the oversized audiobook PR with a stacked series of smaller, reviewable PRs.
|
||||
|
||||
**Architecture:** Preserve the existing `feat/audiobooks` work as the source branch, then create branch cut points that expose one logical layer at a time. Each PR targets the previous branch so reviewers see only the incremental diff.
|
||||
|
||||
**Tech Stack:** Git, GitHub CLI, Go, pnpm/Vite, PostgreSQL migrations.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Close The Oversized PR
|
||||
|
||||
**Files:**
|
||||
- Modify: GitHub PR #17 only.
|
||||
|
||||
- [ ] **Step 1: Post a replacement note**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr comment 17 --repo Silo-Server/silo-server --body 'Closing this oversized PR in favor of a stacked series of smaller PRs. The existing branch is preserved as the source-of-truth while the stack is rebuilt into logical review units.'
|
||||
```
|
||||
|
||||
Expected: GitHub prints the created comment URL.
|
||||
|
||||
- [ ] **Step 2: Close PR #17 without deleting the branch**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr close 17 --repo Silo-Server/silo-server
|
||||
```
|
||||
|
||||
Expected: PR #17 state becomes `CLOSED`; branch `feat/audiobooks` remains available.
|
||||
|
||||
### Task 2: Create Stacked Branches
|
||||
|
||||
**Files:**
|
||||
- Modify: remote Git branches only.
|
||||
|
||||
- [ ] **Step 1: Create branch 1, foundation**
|
||||
|
||||
Branch name: `stack/audiobooks-foundation`
|
||||
|
||||
Scope:
|
||||
- audiobook feature flag/settings
|
||||
- audiobook and podcast schema foundations
|
||||
- scanner support
|
||||
- audiobook media item write path
|
||||
- no native UI
|
||||
- no ABS listener
|
||||
|
||||
- [ ] **Step 2: Create branch 2, native UI MVP**
|
||||
|
||||
Branch name: `stack/audiobooks-native-ui`
|
||||
|
||||
Base: `stack/audiobooks-foundation`
|
||||
|
||||
Scope:
|
||||
- `/api/v1/audiobooks`
|
||||
- audiobook detail/progress endpoints
|
||||
- audiobook frontend route/sidebar/player MVP
|
||||
|
||||
- [ ] **Step 3: Create branch 3, ABS core**
|
||||
|
||||
Branch name: `stack/audiobooks-abs-core`
|
||||
|
||||
Base: `stack/audiobooks-native-ui`
|
||||
|
||||
Scope:
|
||||
- ABS auth/login/refresh/logout
|
||||
- ABS listener and feature-flag listener gating
|
||||
- ABS library/items/play/progress
|
||||
- ABS access control and security hardening
|
||||
|
||||
- [ ] **Step 4: Create branch 4, ABS collections**
|
||||
|
||||
Branch name: `stack/audiobooks-abs-collections`
|
||||
|
||||
Base: `stack/audiobooks-abs-core`
|
||||
|
||||
Scope:
|
||||
- ABS bookmarks
|
||||
- ABS collections
|
||||
- ABS playlists
|
||||
- ABS smart collections
|
||||
- unified collection migration
|
||||
|
||||
- [ ] **Step 5: Create branch 5, extras and polish**
|
||||
|
||||
Branch name: `stack/audiobooks-extras`
|
||||
|
||||
Base: `stack/audiobooks-abs-collections`
|
||||
|
||||
Scope:
|
||||
- podcasts/RSS
|
||||
- stats
|
||||
- author/series extras
|
||||
- catalog audiobook filters/typeahead
|
||||
- performance and enrichment cleanup
|
||||
|
||||
### Task 3: Open The Stacked PRs
|
||||
|
||||
**Files:**
|
||||
- Modify: GitHub PRs only.
|
||||
|
||||
- [ ] **Step 1: Open PR 1**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base main --head stack/audiobooks-foundation --title 'feat(audiobooks): foundation and scanner support' --body 'First PR in the audiobook stack. Adds the schema/settings/scanner foundation without native UI or ABS compatibility.'
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Open PR 2**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-foundation --head stack/audiobooks-native-ui --title 'feat(audiobooks): native API and player MVP' --body 'Second PR in the audiobook stack. Adds native Silo audiobook endpoints and the MVP web player UI.'
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Open PR 3**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-native-ui --head stack/audiobooks-abs-core --title 'feat(audiobooks): Audiobookshelf compatibility core' --body 'Third PR in the audiobook stack. Adds ABS auth, listener, library/items/play/progress, and security hardening.'
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Open PR 4**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-abs-core --head stack/audiobooks-abs-collections --title 'feat(audiobooks): ABS collections, playlists, and smart collections' --body 'Fourth PR in the audiobook stack. Adds ABS bookmarks, collections, playlists, smart collections, and unified collection storage.'
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Open PR 5**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr create --repo Silo-Server/silo-server --base stack/audiobooks-abs-collections --head stack/audiobooks-extras --title 'feat(audiobooks): podcasts, catalog filters, and enrichment polish' --body 'Final PR in the audiobook stack. Adds podcast/RSS extras, audiobook catalog filters/typeahead, and scan/enrichment performance polish.'
|
||||
```
|
||||
|
||||
### Task 4: Verify And Report
|
||||
|
||||
**Files:**
|
||||
- Modify: none.
|
||||
|
||||
- [ ] **Step 1: Verify PR states**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
gh pr list --repo Silo-Server/silo-server --state open --search 'audiobooks in:title' --json number,title,headRefName,baseRefName,url
|
||||
```
|
||||
|
||||
Expected: five open stacked PRs with the base/head chain shown above.
|
||||
|
||||
- [ ] **Step 2: Run branch verification where feasible**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
cd web && pnpm run build
|
||||
```
|
||||
|
||||
Expected: both commands pass on the final stack tip.
|
||||
|
||||
@@ -0,0 +1,617 @@
|
||||
# TheIntroDB read-path correctness (Phase 1 / Layer A) — implementation plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Fix the three read-path correctness gaps in the TheIntroDB provider so it matches the
|
||||
real v3 API contract: (1) honor **TVDB** ids in lookups (today they are silently dropped),
|
||||
(2) capture and use the **real per-segment `confidence`** instead of a hardcoded `0.9`, and
|
||||
(3) when the API returns multiple candidate segments, **pick the most-submitted / highest-confidence**
|
||||
one instead of the first usable. This is Phase 1 of
|
||||
[the marker sources & contribution design](../specs/2026-06-06-marker-sources-and-contribution-design.md);
|
||||
it stands alone, ships value immediately (anime / TheTVDB-first libraries start getting markers),
|
||||
and unblocks the "query-all / best-wins" merge in later phases.
|
||||
|
||||
**Architecture:** All changes are contained within `internal/markers/introdb` plus one added
|
||||
field on `markers.Marker`. The resolver already populates the TVDB id into
|
||||
`Request.ExternalIDs` — only the provider and HTTP client drop it. The HTTP `Client` gains a
|
||||
`tvdbID` argument on `FetchEpisode`/`FetchMovie` (query preference `tmdb → tvdb → imdb`, cache
|
||||
keys extended); the provider reads `ExternalIDKeyTVDB`; `segmentTimestamps` gains `confidence`
|
||||
and `submission_count`; `pickMarker` uses real confidence (with a named default) and ranks
|
||||
candidates. The write path (`BuildUpdatePayload` / `UpsertMarkers`) is **not** touched here.
|
||||
|
||||
**Out of scope (deferred to Phase 2):** threading *per-segment* confidence through
|
||||
`MarkerUpdatePayload` and `UpsertMarkers`. That refactor only becomes observable once
|
||||
`Registry.FetchMerged` can combine segments from different providers; with a single provider
|
||||
today the existing "max confidence across segments" collapse is equivalent, and changing it now
|
||||
would break `TestBuildUpdatePayloadAggregatesConfidence` for no functional gain. It lands with
|
||||
the merge dispatch in Phase 2.
|
||||
|
||||
**Tech Stack:** Go (the `internal/markers/introdb` package is pure Go — no libvips/CGO),
|
||||
PostgreSQL is not involved (no migration), no frontend changes. Tests run in a throwaway Go
|
||||
container (the host has no Go toolchain); a named volume caches module downloads.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
`internal/markers` and `internal/markers/introdb` are pure Go, so no libvips is needed:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
To run a single test, append `-run TestName` and narrow the package path, e.g.
|
||||
`go test ./internal/markers/introdb/ -run TestFetchEpisodeSendsTVDBWhenNoTMDB -v`.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
- `internal/markers/introdb/types.go` — **modify**. Add `Confidence` + `SubmissionCount` to `segmentTimestamps`; add the `defaultConfidence` const.
|
||||
- `internal/markers/types.go` — **modify**. Add `SubmissionCount` to `markers.Marker`.
|
||||
- `internal/markers/introdb/client.go` — **modify**. `tvdbID` arg on `FetchEpisode`/`FetchMovie`; query preference `tmdb → tvdb → imdb`; cache keys include tvdb.
|
||||
- `internal/markers/introdb/provider.go` — **modify**. Read `ExternalIDKeyTVDB` and pass it through; rewrite `pickMarker` for real confidence + best-candidate.
|
||||
- `internal/markers/introdb/client_test.go` — **create**. httptest-based client tests (TVDB query, TMDB preference, caching).
|
||||
- `internal/markers/introdb/provider_test.go` — **create**. End-to-end provider tests (TVDB-only resolves, real/default confidence, best-candidate).
|
||||
|
||||
No migration, no wiring change (`cmd/silo/main.go` constructs the provider but never calls the
|
||||
client directly), no frontend change.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Type plumbing — capture confidence + submission_count
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/types.go`
|
||||
- Modify: `internal/markers/types.go`
|
||||
|
||||
This task is pure struct plumbing (no behavior yet); later tasks' tests exercise it. Verify by
|
||||
compilation.
|
||||
|
||||
- [ ] **Step 1: Add the response fields + default constant in `introdb/types.go`**
|
||||
|
||||
Replace the `Algorithm` const block to add `defaultConfidence` after it:
|
||||
|
||||
```go
|
||||
// Algorithm is the algorithm tag written alongside markers. The version
|
||||
// suffix lets us invalidate or refresh markers if the upstream contract
|
||||
// changes.
|
||||
const Algorithm = "introdb:v3"
|
||||
|
||||
// defaultConfidence is applied when TheIntroDB omits a per-segment confidence
|
||||
// in the /media response. Real per-segment confidence is preferred when present.
|
||||
const defaultConfidence = 0.9
|
||||
```
|
||||
|
||||
Replace the `segmentTimestamps` struct with the version that decodes the two extra fields the
|
||||
v3 `/media` response carries:
|
||||
|
||||
```go
|
||||
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
|
||||
// Either bound may be nil — for intro/recap, start may be omitted (segment
|
||||
// begins at file start); for credits/preview, end may be omitted (segment
|
||||
// runs to file end). Confidence and SubmissionCount are optional per-segment
|
||||
// quality signals used to pick among multiple candidates.
|
||||
type segmentTimestamps struct {
|
||||
StartMs *int64 `json:"start_ms,omitempty"`
|
||||
EndMs *int64 `json:"end_ms,omitempty"`
|
||||
Confidence *float64 `json:"confidence,omitempty"`
|
||||
SubmissionCount *int `json:"submission_count,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add `SubmissionCount` to `markers.Marker`**
|
||||
|
||||
In `internal/markers/types.go`, replace the `Marker` struct:
|
||||
|
||||
```go
|
||||
type Marker struct {
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
SubmissionCount int
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify the packages still compile**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./internal/markers/...
|
||||
```
|
||||
|
||||
Expected: no output, exit 0. (Existing `write_test.go` / `types_test.go` use field-keyed
|
||||
literals, so the added field does not break them.)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/types.go internal/markers/types.go
|
||||
git commit -m "feat(markers): decode introdb per-segment confidence + submission_count"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: TVDB lookups (gap #1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/client.go`
|
||||
- Modify: `internal/markers/introdb/provider.go`
|
||||
- Create: `internal/markers/introdb/client_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing client tests**
|
||||
|
||||
Create `internal/markers/introdb/client_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchEpisodeSendsTVDBWhenNoTMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "55555", "tt1234567", 2, 3, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "55555" {
|
||||
t.Errorf("tvdb_id = %q, want 55555", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "" {
|
||||
t.Errorf("tmdb_id = %q, want empty", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("imdb_id should be omitted when tvdb present, got %q", gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodePrefersTMDBOverTVDBAndIMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "111", "222", "tt333", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "111" {
|
||||
t.Errorf("tmdb_id = %q, want 111", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "" || gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("only tmdb_id expected, got tvdb=%q imdb=%q", gotQuery.Get("tvdb_id"), gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchMovieSendsTVDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"movie"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchMovie(context.Background(), "", "888", "", 0); err != nil {
|
||||
t.Fatalf("FetchMovie: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "888" {
|
||||
t.Errorf("tvdb_id = %q, want 888", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodeCachesByID(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "999", "", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Fatalf("server hits = %d, want 1 (cached after first)", hits)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: FAIL — compile error (`FetchEpisode`/`FetchMovie` still take the old 6/4-arg
|
||||
signatures without `tvdbID`).
|
||||
|
||||
- [ ] **Step 3: Add `tvdbID` to `FetchEpisode` in `client.go`**
|
||||
|
||||
Replace the whole `FetchEpisode` method:
|
||||
|
||||
```go
|
||||
// FetchEpisode looks up segment timestamps for a TV episode.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
if season <= 0 || episode <= 0 {
|
||||
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
q.Set("season", strconv.Itoa(season))
|
||||
q.Set("episode", strconv.Itoa(episode))
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, tvdbID, imdbID, season, episode, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `tvdbID` to `FetchMovie` in `client.go`**
|
||||
|
||||
Replace the whole `FetchMovie` method:
|
||||
|
||||
```go
|
||||
// FetchMovie looks up segment timestamps for a movie.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchMovie(ctx context.Context, tmdbID, tvdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, tvdbID, imdbID, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Extend the cache-key helpers in `client.go`**
|
||||
|
||||
Replace both `cacheKeyEpisode` and `cacheKeyMovie`:
|
||||
|
||||
```go
|
||||
func cacheKeyEpisode(tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:%s:s%de%d:d%d", tvdbID, season, episode, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheKeyMovie(tmdbID, tvdbID, imdbID string, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:movie:%s:d%d", tvdbID, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Read TVDB in the provider and pass it through (`provider.go`)**
|
||||
|
||||
Replace the id-extraction block in `FetchMarkers` (the `tmdbID`/`imdbID` lines and the empty
|
||||
guard):
|
||||
|
||||
```go
|
||||
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
|
||||
tvdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTVDB])
|
||||
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
```
|
||||
|
||||
Then update the two client calls in the `switch req.Kind` block to pass `tvdbID`:
|
||||
|
||||
```go
|
||||
resp, err = p.client.FetchEpisode(ctx, tmdbID, tvdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
|
||||
```
|
||||
```go
|
||||
resp, err = p.client.FetchMovie(ctx, tmdbID, tvdbID, imdbID, durationMS)
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run the client tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestFetch*` tests).
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/client.go internal/markers/introdb/provider.go internal/markers/introdb/client_test.go
|
||||
git commit -m "fix(markers): honor TVDB ids in TheIntroDB lookups"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Real confidence + best-candidate selection (gaps #2, #3)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/provider.go` (`pickMarker`)
|
||||
- Create: `internal/markers/introdb/provider_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing provider tests**
|
||||
|
||||
Create `internal/markers/introdb/provider_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
func newProvider(t *testing.T, body string) *Provider {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
return NewProvider(c)
|
||||
}
|
||||
|
||||
func episodeReq(ids map[string]string) markers.Request {
|
||||
return markers.Request{
|
||||
Kind: markers.ItemKindEpisode,
|
||||
ExternalIDs: ids,
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Duration: 30 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderResolvesTVDBOnly(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, err := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTVDB: "777"}))
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMarkers: %v", err)
|
||||
}
|
||||
if len(res.Markers) != 1 || res.Markers[0].Kind != markers.MarkerKindIntro {
|
||||
t.Fatalf("expected one intro marker, got %+v", res.Markers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderUsesRealConfidence(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000,"confidence":0.42}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if res.Markers[0].Confidence != 0.42 {
|
||||
t.Errorf("confidence = %v, want 0.42 (real value, not hardcoded)", res.Markers[0].Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderDefaultsConfidenceWhenAbsent(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if res.Markers[0].Confidence != defaultConfidence {
|
||||
t.Errorf("confidence = %v, want default %v", res.Markers[0].Confidence, defaultConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPicksMostSubmittedCandidate(t *testing.T) {
|
||||
body := `{"type":"episode","intro":[
|
||||
{"end_ms":50000,"confidence":0.6,"submission_count":2},
|
||||
{"end_ms":61000,"confidence":0.5,"submission_count":9}
|
||||
]}`
|
||||
p := newProvider(t, body)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if got := res.Markers[0].End; got != 61*time.Second {
|
||||
t.Errorf("picked end = %v, want 61s (the submission_count=9 candidate)", got)
|
||||
}
|
||||
if res.Markers[0].SubmissionCount != 9 {
|
||||
t.Errorf("submission_count = %d, want 9", res.Markers[0].SubmissionCount)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: FAIL — `TestProviderUsesRealConfidence` sees `0.9` (hardcoded), `TestProviderPicksMostSubmittedCandidate`
|
||||
gets the first usable (`end=50s`, `SubmissionCount=0`). (`TestProviderResolvesTVDBOnly` already
|
||||
passes from Task 2 — it guards against regression.)
|
||||
|
||||
- [ ] **Step 3: Rewrite `pickMarker` in `provider.go`**
|
||||
|
||||
Replace the entire `pickMarker` function:
|
||||
|
||||
```go
|
||||
// pickMarker selects the best usable segment from a TheIntroDB response array.
|
||||
// `requireEnd` is true for segments where the end timestamp is the load-bearing
|
||||
// field (intro, recap) — they're allowed to start at 0 if `start_ms` is omitted.
|
||||
// For trailing segments (credits, preview) the start is required but the end
|
||||
// defaults to the file duration. When several candidates are usable (e.g. no
|
||||
// duration match narrowed the set), the most-submitted one wins, with higher
|
||||
// confidence breaking ties; with a single candidate this is the previous
|
||||
// first-usable behavior. Real per-segment confidence is used when present,
|
||||
// falling back to defaultConfidence only when the API omits it.
|
||||
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
|
||||
best := markers.Marker{}
|
||||
bestSubs := -1
|
||||
found := false
|
||||
for _, s := range stamps {
|
||||
start := time.Duration(0)
|
||||
end := totalDuration
|
||||
if s.StartMs != nil {
|
||||
start = time.Duration(*s.StartMs) * time.Millisecond
|
||||
}
|
||||
if s.EndMs != nil {
|
||||
end = time.Duration(*s.EndMs) * time.Millisecond
|
||||
}
|
||||
if requireEnd && s.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
if !requireEnd && s.StartMs == nil {
|
||||
continue
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
confidence := defaultConfidence
|
||||
if s.Confidence != nil {
|
||||
confidence = *s.Confidence
|
||||
}
|
||||
subs := 0
|
||||
if s.SubmissionCount != nil {
|
||||
subs = *s.SubmissionCount
|
||||
}
|
||||
if !found || subs > bestSubs || (subs == bestSubs && confidence > best.Confidence) {
|
||||
best = markers.Marker{Kind: kind, Start: start, End: end, Confidence: confidence, SubmissionCount: subs}
|
||||
bestSubs = subs
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestProvider*` tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/provider.go internal/markers/introdb/provider_test.go
|
||||
git commit -m "feat(markers): use real introdb confidence and pick best candidate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Full-package regression + integration build
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Run the entire markers test suite**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
Expected: PASS — the new `introdb` tests plus the existing `write_test.go` /
|
||||
`types_test.go` (unchanged behavior: `BuildUpdatePayload` still aggregates max confidence,
|
||||
`FetchFirstHit` still uses registration order).
|
||||
|
||||
- [ ] **Step 2: Build the binary to confirm nothing downstream broke**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./...
|
||||
```
|
||||
|
||||
Expected: exit 0. (The only callers of the changed client methods are in `provider.go`, updated
|
||||
in Task 2; `cmd/silo/main.go` constructs the provider but never calls the client directly.)
|
||||
|
||||
- [ ] **Step 3: Confirm the working tree is clean**
|
||||
|
||||
```bash
|
||||
git status
|
||||
```
|
||||
|
||||
Expected: clean (all changes committed in Tasks 1–3; this task adds no files).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Gap coverage:** TVDB lookups (Task 2, `TestFetch*` + `TestProviderResolvesTVDBOnly`), real
|
||||
confidence (Task 3, `TestProviderUsesRealConfidence` + `…DefaultsConfidenceWhenAbsent`),
|
||||
best-candidate (Task 3, `TestProviderPicksMostSubmittedCandidate`). Maps to design gaps #1–#3.
|
||||
- **Containment:** the client signature change touches exactly one production caller
|
||||
(`provider.go`); `markers.Marker` is constructed only in `provider.go`. Verified by grep before
|
||||
planning, re-verified by the Task 4 `go build ./...`.
|
||||
- **No behavior regressions:** the write path is untouched, so `BuildUpdatePayload`'s
|
||||
max-confidence aggregation and its test remain valid; per-segment confidence is deferred to
|
||||
Phase 2 where the merge actually consumes it.
|
||||
- **No placeholders:** every code step shows the complete replacement; every run step shows the
|
||||
command and the expected result. No migration, no frontend, no wiring changes.
|
||||
- **Default confidence:** `defaultConfidence = 0.9` preserves today's effective value for
|
||||
responses that omit confidence, so existing stored markers keep equivalent provenance.
|
||||
@@ -0,0 +1,482 @@
|
||||
# Marker sources & TheIntroDB contribution — implementation plan (all phases)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Implement phases in order; each phase is independently shippable.
|
||||
|
||||
**Goal:** Implement the design in
|
||||
[2026-06-06-marker-sources-and-contribution-design.md](../specs/2026-06-06-marker-sources-and-contribution-design.md):
|
||||
fix the TheIntroDB read-path correctness gaps, formalize a multi-source provider model
|
||||
(query-all / best-wins), and add the ability to contribute Silo's own markers back to
|
||||
TheIntroDB — configured **per provider, off by default**. Server-side API + data model only;
|
||||
web UI is a separate follow-up.
|
||||
|
||||
**Architecture:** Five sequential phases. Phase 1 fixes the existing provider (TVDB, real
|
||||
confidence, best-candidate). Phase 2 adds a per-provider config table, per-segment marker
|
||||
provenance, and `Registry.FetchMerged`. Phase 3 adds the submission client, the contribution
|
||||
audit table, and the `ContributionService` engine. Phase 4 exposes the full admin API (manual
|
||||
markers, provider config, contribute, history). Phase 5 adds the daily auto-contribution task.
|
||||
|
||||
**Tech Stack:** Go (chi handlers, pgx repositories), PostgreSQL (paired numbered migrations),
|
||||
no frontend in this plan. Most touched packages (`internal/markers`, `internal/markers/introdb`,
|
||||
`internal/taskmanager/tasks`) are pure Go; `internal/api/handlers` needs libvips (CGO image
|
||||
deps). Tests run in a throwaway Go container (host has no Go toolchain).
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Phase overview & dependencies
|
||||
|
||||
| Phase | Delivers | Depends on | New migrations |
|
||||
|------|----------|-----------|----------------|
|
||||
| 1 | TheIntroDB read-path correctness (TVDB, real confidence, best-candidate) | — | none |
|
||||
| 2 | `marker_provider_config` + per-segment provenance + `FetchMerged` | 1 (real confidence) | `marker_provider_config` |
|
||||
| 3 | Submission client + `marker_contributions` + `ContributionService` | 2 (`Submitter`, provider config) | `marker_contributions` |
|
||||
| 4 | Admin API: manual markers, provider config, contribute, history | 3 (service) | none |
|
||||
| 5 | Daily auto-contribution task | 3 (service), 4 (optional) | none |
|
||||
|
||||
Phase 1's line-level steps are also captured standalone in
|
||||
[2026-06-06-marker-introdb-readpath-correctness.md](2026-06-06-marker-introdb-readpath-correctness.md);
|
||||
they are summarized here so this document is self-contained.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
Pure-Go packages (`internal/markers`, `internal/markers/introdb`, `internal/taskmanager/...`):
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
`internal/api/handlers` (Phase 4) needs libvips:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
sh -c 'apt-get update >/dev/null && apt-get install -y --no-install-recommends libvips-dev >/dev/null && go test ./internal/api/handlers/ -run TestAdminMarkers -v'
|
||||
```
|
||||
|
||||
Whole-project compile: `go build ./...` (same container).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — TheIntroDB read-path correctness
|
||||
|
||||
**Goal:** Honor TVDB ids, capture/use real per-segment confidence + submission_count, and pick
|
||||
the best candidate among multiple. Contained to `internal/markers/introdb` plus one field on
|
||||
`markers.Marker`. No migration, no write-path change.
|
||||
|
||||
**Files:** `internal/markers/introdb/{types.go,client.go,provider.go}` (modify),
|
||||
`internal/markers/types.go` (modify, add `Marker.SubmissionCount`),
|
||||
`internal/markers/introdb/{client_test.go,provider_test.go}` (create).
|
||||
|
||||
### Task 1.1 — Type plumbing
|
||||
- [ ] Add `Confidence *float64` and `SubmissionCount *int` to `segmentTimestamps`
|
||||
(`introdb/types.go`); add `const defaultConfidence = 0.9`.
|
||||
- [ ] Add `SubmissionCount int` to `markers.Marker` (`markers/types.go`).
|
||||
- [ ] `go build ./internal/markers/...` → exit 0. Commit.
|
||||
|
||||
### Task 1.2 — TVDB lookups (gap #1)
|
||||
- [ ] Create `introdb/client_test.go` (httptest): asserts `tvdb_id` sent when only tvdb given,
|
||||
tmdb preferred over tvdb/imdb, movie tvdb, and cache collapses repeat calls. Run → FAIL (compile).
|
||||
- [ ] `client.go`: `FetchEpisode`/`FetchMovie` gain a `tvdbID` arg; query preference
|
||||
`tmdb → tvdb → imdb` via a `switch`; the empty-id guard includes tvdb; `cacheKeyEpisode`/
|
||||
`cacheKeyMovie` include a `tvdb:` branch.
|
||||
- [ ] `provider.go` `FetchMarkers`: read `req.ExternalIDs[markers.ExternalIDKeyTVDB]`, include it
|
||||
in the empty guard, pass it to both client calls.
|
||||
- [ ] Run client tests → PASS. Commit.
|
||||
|
||||
### Task 1.3 — Real confidence + best-candidate (gaps #2, #3)
|
||||
- [ ] Create `introdb/provider_test.go`: TVDB-only resolves; real confidence used; default when
|
||||
absent; most-submitted candidate wins (ties broken by confidence). Run → FAIL.
|
||||
- [ ] Rewrite `pickMarker` to use real confidence (fallback `defaultConfidence`), set
|
||||
`SubmissionCount`, and select the candidate with the highest `(submission_count, confidence)`.
|
||||
- [ ] Run → PASS. Commit.
|
||||
|
||||
### Task 1.4 — Regression
|
||||
- [ ] `go test ./internal/markers/... && go build ./...` → all green (existing `write_test.go`/
|
||||
`types_test.go` unaffected; write path untouched). Commit if needed.
|
||||
|
||||
(Full code for every step is in the standalone Phase 1 plan linked above.)
|
||||
|
||||
---
|
||||
|
||||
# Phase 2 — Per-provider config, per-segment provenance, FetchMerged
|
||||
|
||||
**Goal:** Add the `marker_provider_config` table that drives which providers are queried (and,
|
||||
later, submitted to); make the marker write path carry **per-segment** provider/confidence/
|
||||
algorithm so a merged multi-provider result writes correct provenance; add
|
||||
`Registry.FetchMerged` (query all enabled providers, keep the best candidate per segment) and
|
||||
switch the lazy-playback path to it. With only TheIntroDB enabled, behavior is unchanged.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/<next>_marker_provider_config.{up,down}.sql`
|
||||
- Create: `internal/markers/provider_config.go`, `internal/markers/provider_config_test.go`
|
||||
- Modify: `internal/markers/types.go` (`Marker.ProviderID`/`Algorithm`; `Registry.FetchMerged`)
|
||||
- Modify: `internal/markers/write.go` (per-segment provenance in `MarkerUpdatePayload`/`BuildUpdatePayload`)
|
||||
- Modify: `internal/scanner/file_repo.go` (`MarkerUpdate` + `applySegmentPatch` per-segment provenance)
|
||||
- Modify: the `markers.MarkerUpdatePayload` → `scanner.MarkerUpdate` conversion (in the lazy-markers handler/adapter)
|
||||
- Modify: `internal/api/handlers/playback_lazy_markers.go` (call `FetchMerged`)
|
||||
- Modify: `cmd/silo/main.go` (construct the config store; pass to the registry)
|
||||
|
||||
### Task 2.1 — Migration: `marker_provider_config`
|
||||
- [ ] Write `migrations/<next>_marker_provider_config.up.sql` (use the next free number; latest is 180):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY,
|
||||
fetch_enabled boolean NOT NULL DEFAULT true,
|
||||
fetch_priority integer NOT NULL DEFAULT 100,
|
||||
contribute_enabled boolean NOT NULL DEFAULT false,
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false,
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: `DROP TABLE public.marker_provider_config;`
|
||||
- [ ] Commit.
|
||||
|
||||
### Task 2.2 — Provider config store
|
||||
- [ ] Create `internal/markers/provider_config.go`:
|
||||
|
||||
```go
|
||||
type ProviderConfig struct {
|
||||
Provider string
|
||||
FetchEnabled bool
|
||||
FetchPriority int
|
||||
ContributeEnabled bool
|
||||
ContributeAutoLocal bool
|
||||
ContributeMinConfidence float64
|
||||
}
|
||||
|
||||
// ProviderConfigStore is a cached read/write façade over marker_provider_config.
|
||||
// Reads serve from an in-memory snapshot; Update writes through and refreshes.
|
||||
type ProviderConfigStore struct { /* pool, mu, cache map[string]ProviderConfig */ }
|
||||
|
||||
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore
|
||||
func (s *ProviderConfigStore) Reload(ctx context.Context) error // load all rows into cache
|
||||
func (s *ProviderConfigStore) List() []ProviderConfig // snapshot copy
|
||||
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool)
|
||||
func (s *ProviderConfigStore) Update(ctx context.Context, p ProviderConfig) error // upsert + refresh
|
||||
// EnabledForFetch returns fetch_enabled providers sorted by (fetch_priority asc, provider asc).
|
||||
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig
|
||||
```
|
||||
|
||||
- [ ] Hot-reload: subscribe the store's `Reload` to the existing `cache.EventSettingsChanged`
|
||||
Redis channel (mirror the `introdb.api_key` reload wiring in `cmd/silo/main.go`), or add a
|
||||
dedicated `marker_provider_config_changed` event published by `Update`. Either is acceptable;
|
||||
document which.
|
||||
- [ ] `provider_config_test.go`: with a real/pgxmock pool or an injected fake, assert
|
||||
`EnabledForFetch` filters disabled rows and sorts by priority. Commit.
|
||||
|
||||
### Task 2.3 — Per-segment marker provenance (write path)
|
||||
|
||||
This makes a merged result (intro from provider A, credits from provider B) persist the correct
|
||||
per-segment `*_markers_provider` / `*_markers_confidence` / `*_markers_algorithm`. The DB columns
|
||||
already exist; today `applySegmentPatch` applies one shared triple.
|
||||
|
||||
- [ ] Add to `markers.Marker`: `ProviderID string`, `Algorithm string` (so each segment carries
|
||||
its origin). `provider.go` (Phase 1) sets these from `ProviderID`/`Algorithm` per marker.
|
||||
- [ ] Rework `markers.MarkerUpdatePayload` (`write.go`) from one shared `Confidence`/`Provider`/
|
||||
`Algorithm` to a per-segment struct, e.g.:
|
||||
|
||||
```go
|
||||
type SegmentPayload struct { Start, End *float64; Provider *string; Confidence *float64; Algorithm string }
|
||||
type MarkerUpdatePayload struct {
|
||||
Intro, Credits, Recap, Preview SegmentPayload
|
||||
Source string // shared source class (e.g. "online"); per-kind provider differs
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Update `BuildUpdatePayload` to fill each `SegmentPayload` from the matching `Marker`
|
||||
(provider/confidence/algorithm per segment). Update `write_test.go`
|
||||
(`TestBuildUpdatePayloadAggregatesConfidence` → assert per-segment confidence: intro 0.7,
|
||||
credits 0.9, recap 0.5).
|
||||
- [ ] Read `applySegmentPatch` and the `MarkerUpdatePayload → scanner.MarkerUpdate` conversion
|
||||
first, then extend `scanner.MarkerUpdate` + `applySegmentPatch` so each segment's
|
||||
provider/confidence/algorithm come from that segment (not a shared field). `nextSharedMarkerAttribution`
|
||||
keeps populating the legacy shared `markers_source`/`markers_confidence` columns from the
|
||||
highest-confidence applied segment.
|
||||
- [ ] `go test ./internal/markers/... ./internal/scanner/...` → green. Commit.
|
||||
|
||||
### Task 2.4 — `Registry.FetchMerged`
|
||||
- [ ] Give `Registry` access to the enabled set (constructor takes a `ProviderConfigStore`, or
|
||||
`FetchMerged` takes an `[]ProviderConfig` snapshot). Add:
|
||||
|
||||
```go
|
||||
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
|
||||
// segment kind, the candidate with the highest (SubmissionCount, Confidence),
|
||||
// breaking ties by the provider's fetch_priority. Per-segment ProviderID/
|
||||
// Algorithm are preserved so the write path records correct provenance.
|
||||
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error)
|
||||
```
|
||||
|
||||
Algorithm: fan out via goroutines (bounded) to enabled providers; collect `(ProviderConfig,
|
||||
Result, error)`; for each `MarkerKind`, choose the best `Marker` across results; assemble a
|
||||
`Result` whose `Markers` carry their winning provider's `ProviderID`/`Algorithm`. Log and skip
|
||||
provider errors (like `FetchFirstHit`). Return `ok=false` if no segment found.
|
||||
|
||||
- [ ] `types_test.go`: add a merge test — two fake providers, one wins intro, the other wins
|
||||
credits by submission_count; assert the merged result has both with correct per-segment
|
||||
ProviderID. Keep the existing `FetchFirstHit` tests. Commit.
|
||||
|
||||
### Task 2.5 — Switch lazy path + wire store
|
||||
- [ ] In `internal/api/handlers/playback_lazy_markers.go`, replace the `FetchFirstHit` call in
|
||||
`fetchOnlineMarkersForPlayback` with `FetchMerged`.
|
||||
- [ ] In `cmd/silo/main.go`, construct `NewProviderConfigStore`, `Reload` it at startup, wire its
|
||||
hot-reload, and pass it to the registry. (TheIntroDB stays registered as today.)
|
||||
- [ ] `go build ./... && go test ./internal/markers/...` → green. With only `introdb` enabled,
|
||||
`FetchMerged` returns the same result `FetchFirstHit` did — verify a provider test asserting
|
||||
single-provider parity. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 3 — Submission client, contribution tracking, service engine
|
||||
|
||||
**Goal:** Build the machinery to submit a marker to a provider, idempotently and audited —
|
||||
without yet wiring a trigger. Adds the `Submitter` capability, the introdb submission/stats
|
||||
client methods, the `marker_contributions` table, and the `ContributionService`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/types.go` (`Submitter`, `SubmissionRequest`/`SubmissionResult`/`UserStats`)
|
||||
- Modify: `internal/markers/introdb/{types.go,client.go,provider.go}` (submit + stats; implement `Submitter`)
|
||||
- Create: `migrations/<next>_marker_contributions.{up,down}.sql`
|
||||
- Create: `internal/markers/contribution_repo.go`, `internal/markers/contribute.go` (+ tests)
|
||||
- Modify: `cmd/silo/main.go` (construct the service)
|
||||
|
||||
### Task 3.1 — `Submitter` capability + DTOs
|
||||
- [ ] Add to `internal/markers/types.go`:
|
||||
|
||||
```go
|
||||
type SubmissionRequest struct {
|
||||
Kind ItemKind
|
||||
ExternalIDs map[string]string // tmdb/imdb (tvdb not accepted by /submit; tmdb required)
|
||||
SeasonNumber int
|
||||
EpisodeNumber int
|
||||
Segment MarkerKind
|
||||
Start, End *time.Duration // nil start ok for intro/recap; nil end ok for credits/preview
|
||||
Duration time.Duration
|
||||
}
|
||||
type SubmissionResult struct { ID string; Status string; Weight float64 } // status: pending|accepted|rejected
|
||||
type UserStats struct { Total, Accepted, Pending, Rejected int; AcceptanceRate float64; CurrentStreak, BestStreak int }
|
||||
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Commit (compiles; no implementer yet).
|
||||
|
||||
### Task 3.2 — introdb submission client
|
||||
- [ ] `introdb/types.go`: add the request/response shapes for `POST /v3/submit`
|
||||
(`tmdb_id`,`imdb_id`,`type`,`segment`,`season`,`episode`,`video_duration_ms`,`start_ms`,`end_ms`;
|
||||
response `{submissions:[{id,status,weight,...}]}`) and `GET /v3/user/stats`. Remove the
|
||||
"Submissions are intentionally not supported" package note.
|
||||
- [ ] `introdb/client.go`: add `SubmitSegment(ctx, ...) (*submitResponse, error)` (POST,
|
||||
`Authorization: Bearer` **required** — return an error if `apiKey == ""`; honor
|
||||
`X-UsageLimit-Reset` on 429) and `FetchUserStats(ctx) (*userStats, error)` (GET).
|
||||
- [ ] `introdb/provider.go`: implement `markers.Submitter` — map `SubmissionRequest` to the
|
||||
client call (apply the v3 null rules: intro/recap drop a zero start to `null`; credits/preview
|
||||
send `null` end when absent), translate `SubmissionResult`/`UserStats`.
|
||||
- [ ] `client_test.go`/`provider_test.go`: httptest asserts the submit body shape, bearer header,
|
||||
and that an empty key errors before any HTTP call; stats parse. Commit.
|
||||
|
||||
### Task 3.3 — `marker_contributions` table + repo
|
||||
- [ ] Migration `<next>_marker_contributions.up.sql` (per design §C4):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL,
|
||||
segment_kind text NOT NULL,
|
||||
source text NOT NULL,
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL,
|
||||
submission_id uuid,
|
||||
status text NOT NULL,
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: drop the table.
|
||||
- [ ] `internal/markers/contribution_repo.go`: `ContributionStore` with `AlreadySubmitted(ctx,
|
||||
fileID, provider, kind, contentHash) (bool, error)`, `Record(ctx, ContributionRow) error`
|
||||
(upsert on the unique key), `ListByFile(ctx, fileID) ([]ContributionRow, error)`. Add a
|
||||
`ContentHash(kind string, startMs, endMs, durationMs *int64) string` helper (stable SHA-256).
|
||||
- [ ] Repo test (value-hash idempotency, list). Commit.
|
||||
|
||||
### Task 3.4 — `ContributionService`
|
||||
- [ ] `internal/markers/contribute.go`:
|
||||
|
||||
```go
|
||||
type ContributionService struct { /* registry, resolver, cfg *ProviderConfigStore, store *ContributionStore, log */ }
|
||||
|
||||
// ContributeFile submits the file's eligible markers to enabled submitter providers.
|
||||
// opts may scope to one provider and/or specific segment kinds. Auto callers pass
|
||||
// requireAutoLocal=true so only providers with contribute_auto_local participate.
|
||||
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error)
|
||||
```
|
||||
|
||||
Per provider (from `cfg.List()` filtered to `Submitter` + `contribute_enabled`, and
|
||||
`contribute_auto_local` when `opts.Auto`):
|
||||
1. **Eligibility (C2):** for each requested segment present on the file, require
|
||||
`source ∈ {scanner, manual}` (skip `online` — circular); resolve external IDs via the existing
|
||||
resolver (skip if none / tmdb required by `/submit`); for auto require `source=scanner`,
|
||||
`kind=intro`, `confidence ≥ contribute_min_confidence`.
|
||||
2. **Idempotency:** compute `ContentHash`; skip if `AlreadySubmitted`.
|
||||
3. **Submit:** `provider.(Submitter).SubmitMarker(...)`.
|
||||
4. **Persist:** `store.Record(...)` with returned status/submission_id (or `status="error"`).
|
||||
|
||||
- [ ] `contribute_test.go` with a fake `Submitter` + in-memory store: asserts online-sourced
|
||||
markers are skipped, sub-threshold auto markers skipped, duplicates skipped, a fresh manual
|
||||
marker submitted and recorded. Commit.
|
||||
|
||||
### Task 3.5 — Wire
|
||||
- [ ] `cmd/silo/main.go`: construct `ContributionService` (registry, resolver, config store,
|
||||
contribution store). Not yet called by any handler/task. `go build ./...` + tests green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 4 — Admin API (manual markers, provider config, contribute, history)
|
||||
|
||||
**Goal:** Expose the full backend contract under `RequireAdmin`. Mirror the structure of
|
||||
`internal/api/handlers/admin_intro.go` (interface-typed deps, `chi.URLParam`, `writeError`/
|
||||
`writeJSON`, `MarkerUpdateNotifier` after writes).
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/api/handlers/admin_markers.go` (+ test), `internal/api/handlers/admin_marker_providers.go` (+ test)
|
||||
- Modify: `internal/api/router.go` (register routes in the `RequireAdmin` group)
|
||||
- Modify: `cmd/silo/main.go` (construct + inject the handlers)
|
||||
|
||||
### Task 4.1 — Manual marker endpoints (design §C3)
|
||||
- [ ] `admin_markers.go`:
|
||||
- `GET /admin/files/{fileId}/markers` → current per-segment values + provenance
|
||||
(`source`,`provider`,`confidence`,`algorithm`,`detected_at`) for intro/recap/credits/preview.
|
||||
- `PUT /admin/files/{fileId}/markers` → body per segment `{start,end}` (seconds) or `null` to
|
||||
clear; write `source = "manual"` via `scanner.FileRepository.UpsertMarkers` (priority 4);
|
||||
then `MarkerUpdateNotifier.MarkersUpdated(file)`. Validation: `end>start`; intro/recap may
|
||||
omit start; credits/preview may omit end; within `[0, duration]`.
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` → clear one segment.
|
||||
- `GET|PUT /admin/items/{id}/markers` → resolve to the item's primary file (convenience).
|
||||
- [ ] Handler test (libvips container): PUT writes manual markers and returns provenance;
|
||||
invalid range → 400; clear nulls the segment. Commit.
|
||||
|
||||
### Task 4.2 — Provider config + validate endpoints (design §C7)
|
||||
- [ ] `admin_marker_providers.go`:
|
||||
- `GET /admin/markers/providers` → for each registered provider: id, `isSubmitter`, its
|
||||
`marker_provider_config` row, and (best-effort cached) `UserStats` for submitters with a key.
|
||||
- `PUT /admin/markers/providers/{provider}` → update the config row (validated; unknown → 404).
|
||||
- `POST /admin/markers/providers/{provider}/validate` → `Submitter.FetchUserStats`; non-submitter → 400.
|
||||
- [ ] Test: list returns introdb with its config; update flips `contribute_enabled`; validate on
|
||||
a non-submitter → 400. Commit.
|
||||
|
||||
### Task 4.3 — Contribution endpoints (design §C7)
|
||||
- [ ] In `admin_markers.go` (or a small `admin_contributions.go`):
|
||||
- `POST /admin/files/{fileId}/contribute` (optional body `{provider?, segments?[]}`) →
|
||||
`ContributionService.ContributeFile(...)`; returns per (provider, segment) outcome.
|
||||
- `GET /admin/files/{fileId}/contributions` → `ContributionStore.ListByFile`.
|
||||
- [ ] Test with a fake service/store. Commit.
|
||||
|
||||
### Task 4.4 — Routing + wiring
|
||||
- [ ] Register all new routes inside the existing `RequireAdmin` group in `internal/api/router.go`
|
||||
(mirror the `redetect-intro` registration). Inject the handlers in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/api/handlers/ -run 'TestAdminMarkers|TestAdminMarkerProviders'`
|
||||
(libvips container) → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 5 — Daily auto-contribution task
|
||||
|
||||
**Goal:** A scheduled task that, for each provider with `contribute_enabled &&
|
||||
contribute_auto_local`, submits high-confidence local intro detections that haven't been
|
||||
contributed. Mirror `internal/taskmanager/tasks/detect_intro_markers.go`.
|
||||
|
||||
**Files:** Create `internal/taskmanager/tasks/contribute_markers.go` (+ test); modify
|
||||
`cmd/silo/main.go` (register the task).
|
||||
|
||||
### Task 5.1 — `ContributeMarkersTask`
|
||||
- [ ] Implement the `taskmanager.Task` interface (`Key="contribute_markers"`, `Name`,
|
||||
`Description`, `Category=TaskCategoryLibrary`, `IsHidden() bool`, `DefaultTriggers()` daily e.g.
|
||||
`"04:00"` — after the 03:30 detection task, `Execute`).
|
||||
- [ ] `Execute`: short-circuit if no provider has `contribute_enabled && contribute_auto_local`
|
||||
(read `ProviderConfigStore`). Otherwise page through episode files with
|
||||
`intro_markers_source='scanner'` and `intro_markers_confidence ≥` the provider's
|
||||
`contribute_min_confidence` (a new `ContributionStore`/repo query, batched), and call
|
||||
`ContributionService.ContributeFile(file, ContributeOptions{Auto:true})`. Report progress; obey
|
||||
the introdb usage limit (the client already backs off on `X-UsageLimit-Reset`); idempotency
|
||||
makes interrupted runs resumable.
|
||||
- [ ] Test with fakes: enabled+auto submits eligible files; disabled → no-op; sub-threshold
|
||||
skipped. Commit.
|
||||
|
||||
### Task 5.2 — Register
|
||||
- [ ] Register the task where `DetectIntroMarkersTask` is registered in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/taskmanager/...` → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
## Final integration verification
|
||||
|
||||
**Files:** none (build + runtime).
|
||||
|
||||
- [ ] **Build the image** (compiles backend + frontend):
|
||||
```bash
|
||||
docker build --build-arg BUILD_REVISION=$(git rev-parse --short HEAD) --build-arg BUILD_DIRTY=false -t silo-server:markers-test .
|
||||
```
|
||||
- [ ] **Recreate the container** (applies the new migrations on startup) and wait for health:
|
||||
```bash
|
||||
docker compose up -d silo
|
||||
for i in $(seq 1 24); do s=$(docker inspect -f '{{.State.Health.Status}}' silo-silo-1); echo "$s"; [ "$s" = healthy ] && break; sleep 5; done
|
||||
```
|
||||
- [ ] **Verify migrations applied + tables exist:**
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT to_regclass('public.marker_provider_config'), to_regclass('public.marker_contributions');"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c "SELECT * FROM marker_provider_config;"
|
||||
```
|
||||
Expected: both tables present; an `introdb` row with `fetch_enabled=t, contribute_enabled=f`.
|
||||
- [ ] **Smoke the read path (TVDB):** trigger playback (or call the marker fetch) for a
|
||||
TVDB-only-matched episode and confirm markers now populate where they previously did not.
|
||||
- [ ] **Smoke contribution (manual, dry):** enable contribution for introdb against a **test/staging**
|
||||
TheIntroDB key, `PUT` a manual marker on a file, `POST /admin/files/{id}/contribute`, then
|
||||
`GET /admin/files/{id}/contributions` shows a `pending` row; a second contribute is idempotent
|
||||
(no new row). Use a non-production key so test data isn't submitted to the real database.
|
||||
- [ ] `git status` → clean (all changes committed per phase).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Phase independence:** each phase compiles, tests, and ships alone; with only `introdb`
|
||||
enabled and contribution off (defaults), Phases 2–5 are observationally inert until an operator
|
||||
enables a provider — so they can land ahead of any behavior change.
|
||||
- **Per-provider, off by default:** `contribute_enabled`/`contribute_auto_local` default `false`
|
||||
in the migration; the service refuses to submit otherwise; auto additionally gated on
|
||||
`contribute_auto_local`. The `provider` column on `marker_contributions` and the per-provider
|
||||
loop make multi-target submission first-class.
|
||||
- **Circular-contribution guard:** `source='online'` is never contributed (C2 eligibility) — has
|
||||
a dedicated unit test in Task 3.4.
|
||||
- **Idempotency:** value `content_hash` + the unique constraint mean repeat/interrupted runs
|
||||
don't double-submit, while a corrected value (new hash) does submit — matching TheIntroDB's
|
||||
weighted-average model.
|
||||
- **Write-path accuracy:** Task 2.3 requires reading `applySegmentPatch` and the
|
||||
`MarkerUpdatePayload → scanner.MarkerUpdate` conversion before editing; the per-segment columns
|
||||
already exist in `media_files`, so this is a plumbing change, not a schema change.
|
||||
- **Migration numbers:** `marker_provider_config` and `marker_contributions` take the next two
|
||||
free numbers at implementation time (181+; latest on disk is 180). Re-check for collisions
|
||||
before writing.
|
||||
- **Out of scope (tracked in the design doc):** all web UI; per-user TheIntroDB accounts; a
|
||||
`marker_provider.v1` plugin capability. Clean seams left for each.
|
||||
@@ -0,0 +1,301 @@
|
||||
# Ebook Reader Full Parity Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Bring the core ebook reader up to the personal reader feature set with server-persisted config, annotations/bookmarks, selection tools, reading aids, and advanced settings.
|
||||
|
||||
**Architecture:** Keep ebook reading in core, matching audiobooks: core owns the ebook media type and reader behavior; metadata remains plugin-provided. Extend the existing `/ebooks/{content_id}` reader API and `EbookReaderHandler` rather than creating a separate plugin-style reader subsystem. Persist per-user/per-profile/per-book state in Postgres so settings, annotations, and bookmarks roam across devices.
|
||||
|
||||
**Tech Stack:** Go/Chi handlers, pgx/Postgres migrations, React 19, TanStack Query, Foliate/readest reader wrapper, Vitest, Go unit tests.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- `migrations/sql/20260608000300_ebook_reader_state.sql`
|
||||
- Creates `ebook_reader_config` and `ebook_reader_annotations`.
|
||||
- `internal/api/handlers/ebook_reader.go`
|
||||
- Adds reader config and annotation types, store interfaces, handlers, validation, and Postgres store methods.
|
||||
- `internal/api/handlers/ebook_reader_test.go`
|
||||
- Adds handler tests for config, annotation CRUD, access scoping, and validation.
|
||||
- `internal/api/router.go`
|
||||
- Wires config/annotation stores and routes under `/ebooks/{content_id}`.
|
||||
- `web/src/reader/FoliateBookReader.tsx`
|
||||
- Adds selection/annotation/readable-text/content-popup reader handles and applies saved config.
|
||||
- `web/src/pages/EbookReader.tsx`
|
||||
- Adds server-backed settings, annotation/bookmark UI, selection tools, TTS controls, wake-lock/e-ink controls, and advanced reader controls.
|
||||
- `web/src/hooks/useTTS.ts`, `web/src/hooks/useScreenWakeLock.ts`, `web/src/hooks/useEinkMode.ts`
|
||||
- Port focused personal-reader hooks into core web.
|
||||
- `web/src/reader/ebookReaderApi.ts`
|
||||
- Centralizes reader config and annotation API calls for tests and page usage.
|
||||
- `web/src/pages/EbookReader.test.tsx`, `web/src/reader/FoliateBookReader.test.ts`
|
||||
- Adds UI and wrapper tests.
|
||||
|
||||
## Task 1: Server-Persisted Reader Config
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/sql/20260608000300_ebook_reader_state.sql`
|
||||
- Modify: `internal/api/handlers/ebook_reader.go`
|
||||
- Modify: `internal/api/handlers/ebook_reader_test.go`
|
||||
- Modify: `internal/api/router.go`
|
||||
- Create: `web/src/reader/ebookReaderApi.ts`
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
- Modify: `web/src/pages/EbookReader.test.tsx`
|
||||
|
||||
- [ ] **Step 1: Write failing Go handler tests**
|
||||
|
||||
Add tests that:
|
||||
- `GET /ebooks/{content_id}/reader-config` returns `{ "config": {} }` when no row exists.
|
||||
- `PUT /ebooks/{content_id}/reader-config` saves JSON config for the authenticated user/profile/content.
|
||||
- inaccessible content returns 404 through the existing `ItemAccess` path.
|
||||
- invalid non-object config returns 400.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/api/handlers -run 'TestEbookReader.*Config' -count=1
|
||||
```
|
||||
|
||||
Expected: fail because config store/handlers do not exist.
|
||||
|
||||
- [ ] **Step 2: Add migration**
|
||||
|
||||
Create `ebook_reader_config`:
|
||||
|
||||
```sql
|
||||
CREATE TABLE ebook_reader_config (
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
profile_id TEXT NOT NULL,
|
||||
content_id TEXT NOT NULL REFERENCES media_items(content_id) ON DELETE CASCADE,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (user_id, profile_id, content_id)
|
||||
);
|
||||
|
||||
CREATE INDEX ebook_reader_config_profile_updated
|
||||
ON ebook_reader_config (user_id, profile_id, updated_at DESC);
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Implement config store and handlers**
|
||||
|
||||
Add `EbookReaderConfig`, `EbookReaderConfigStore`, `HandleGetConfig`, `HandleSaveConfig`, and `PGEbookReaderConfigStore` methods in `ebook_reader.go`.
|
||||
|
||||
Validation:
|
||||
- authenticated user required
|
||||
- `content_id` required
|
||||
- `config` must decode to a JSON object
|
||||
- content access checked with `FileAuthorizer.ItemAccess.EnsureAccessible`
|
||||
|
||||
- [ ] **Step 4: Wire routes and store**
|
||||
|
||||
In `internal/api/router.go`, create the store when `deps.DB != nil`, assign it to `ebookReaderHandler`, and add:
|
||||
|
||||
```go
|
||||
r.Get("/{content_id}/reader-config", ebookReaderHandler.HandleGetConfig)
|
||||
r.Put("/{content_id}/reader-config", ebookReaderHandler.HandleSaveConfig)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Add web API wrapper**
|
||||
|
||||
Create `web/src/reader/ebookReaderApi.ts` with:
|
||||
|
||||
```ts
|
||||
export function ebookReaderConfigPath(contentID: string): string
|
||||
export async function fetchEbookReaderConfig(contentID: string): Promise<Record<string, unknown>>
|
||||
export async function saveEbookReaderConfig(contentID: string, config: Record<string, unknown>): Promise<Record<string, unknown>>
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Move settings persistence from localStorage to server**
|
||||
|
||||
Update `EbookReader.tsx` to fetch reader config on load and save settings through the config endpoint. Keep a local fallback only until the server value arrives.
|
||||
|
||||
- [ ] **Step 7: Verify and commit**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/api/handlers -run 'TestEbookReader.*Config' -count=1
|
||||
cd web && pnpm test src/pages/EbookReader.test.tsx src/reader/FoliateBookReader.test.ts --run
|
||||
cd web && pnpm build
|
||||
```
|
||||
|
||||
Commit:
|
||||
|
||||
```bash
|
||||
git add migrations/sql/20260608000300_ebook_reader_state.sql internal/api/handlers/ebook_reader.go internal/api/handlers/ebook_reader_test.go internal/api/router.go web/src/reader/ebookReaderApi.ts web/src/pages/EbookReader.tsx web/src/pages/EbookReader.test.tsx
|
||||
git commit -m "feat: persist ebook reader config"
|
||||
```
|
||||
|
||||
## Task 2: Server-Persisted Annotations And Bookmarks
|
||||
|
||||
**Files:**
|
||||
- Modify: `migrations/sql/20260608000300_ebook_reader_state.sql`
|
||||
- Modify: `internal/api/handlers/ebook_reader.go`
|
||||
- Modify: `internal/api/handlers/ebook_reader_test.go`
|
||||
- Modify: `internal/api/router.go`
|
||||
- Modify: `web/src/reader/ebookReaderApi.ts`
|
||||
- Modify: `web/src/reader/FoliateBookReader.tsx`
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
|
||||
- [ ] **Step 1: Write failing annotation handler tests**
|
||||
|
||||
Cover list, create, update, delete. Test annotation fields:
|
||||
|
||||
```json
|
||||
{
|
||||
"kind": "highlight",
|
||||
"cfi_range": "epubcfi(/6/4,/1:0,/1:12)",
|
||||
"selected_text": "sample text",
|
||||
"note": "note text",
|
||||
"style": "highlight",
|
||||
"color": "#facc15"
|
||||
}
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/api/handlers -run 'TestEbookReader.*Annotation|TestEbookReader.*Bookmark' -count=1
|
||||
```
|
||||
|
||||
Expected: fail because annotation APIs do not exist.
|
||||
|
||||
- [ ] **Step 2: Add annotation table**
|
||||
|
||||
Extend migration with:
|
||||
|
||||
```sql
|
||||
CREATE TABLE ebook_reader_annotations (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
profile_id TEXT NOT NULL,
|
||||
content_id TEXT NOT NULL REFERENCES media_items(content_id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('highlight', 'note', 'bookmark')),
|
||||
cfi_range TEXT,
|
||||
location TEXT,
|
||||
selected_text TEXT NOT NULL DEFAULT '',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
style TEXT NOT NULL DEFAULT 'highlight',
|
||||
color TEXT NOT NULL DEFAULT '#facc15',
|
||||
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
CHECK ((kind = 'bookmark' AND location IS NOT NULL) OR (kind <> 'bookmark' AND cfi_range IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE INDEX ebook_reader_annotations_book
|
||||
ON ebook_reader_annotations (user_id, profile_id, content_id, updated_at DESC);
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Implement annotation store and handlers**
|
||||
|
||||
Add handlers:
|
||||
- `GET /ebooks/{content_id}/annotations`
|
||||
- `POST /ebooks/{content_id}/annotations`
|
||||
- `PATCH /ebooks/{content_id}/annotations/{annotation_id}`
|
||||
- `DELETE /ebooks/{content_id}/annotations/{annotation_id}`
|
||||
|
||||
- [ ] **Step 4: Wire Foliate annotation drawing and selection**
|
||||
|
||||
Extend `FoliateBookReaderHandle` with:
|
||||
- `createSelectionAnnotation()`
|
||||
- `clearSelection()`
|
||||
|
||||
Draw stored annotations using Foliate `addAnnotation`.
|
||||
|
||||
- [ ] **Step 5: Add reader UI**
|
||||
|
||||
Add:
|
||||
- selection popover
|
||||
- highlight button
|
||||
- note button
|
||||
- bookmark button
|
||||
- annotation list in the side panel
|
||||
- delete/update note controls
|
||||
|
||||
- [ ] **Step 6: Verify and commit**
|
||||
|
||||
Run handler tests, web tests, and build. Commit:
|
||||
|
||||
```bash
|
||||
git commit -m "feat: add ebook annotations and bookmarks"
|
||||
```
|
||||
|
||||
## Task 3: Reader Tools, TTS, And Reading Aids
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/hooks/useTTS.ts`
|
||||
- Create: `web/src/hooks/useScreenWakeLock.ts`
|
||||
- Create: `web/src/hooks/useEinkMode.ts`
|
||||
- Modify: `web/src/reader/FoliateBookReader.tsx`
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
- Modify: `web/src/pages/EbookReader.test.tsx`
|
||||
|
||||
- [ ] **Step 1: Port focused hooks from personal reader**
|
||||
|
||||
Bring over:
|
||||
- Web Speech TTS controller
|
||||
- screen wake lock hook
|
||||
- e-ink body-class hook
|
||||
|
||||
- [ ] **Step 2: Add readable text handle**
|
||||
|
||||
Expose `getReadableText()` from `FoliateBookReader` for TTS.
|
||||
|
||||
- [ ] **Step 3: Add UI controls**
|
||||
|
||||
Add side-panel controls for:
|
||||
- Speak current text
|
||||
- Pause/resume/stop
|
||||
- voice/rate/pitch
|
||||
- wake lock
|
||||
- e-ink mode
|
||||
|
||||
- [ ] **Step 4: Add content popups and helpers**
|
||||
|
||||
Add selection popover actions:
|
||||
- define
|
||||
- translate
|
||||
|
||||
Add reader content popup handling for footnotes/images/tables when Foliate emits them.
|
||||
|
||||
- [ ] **Step 5: Verify and commit**
|
||||
|
||||
Run focused web tests and build. Commit:
|
||||
|
||||
```bash
|
||||
git commit -m "feat: add ebook reader tools and aids"
|
||||
```
|
||||
|
||||
## Task 4: Advanced Reader Settings
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/reader/FoliateBookReader.tsx`
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
- Modify: `web/src/pages/EbookReader.test.tsx`
|
||||
|
||||
- [ ] **Step 1: Add advanced reader settings**
|
||||
|
||||
Add server-persisted settings for:
|
||||
- RTL
|
||||
- writing mode
|
||||
- zoom/scale
|
||||
- brightness
|
||||
- hyphenation toggle
|
||||
- custom font selection/upload if API support is added in a later task
|
||||
|
||||
- [ ] **Step 2: Verify and commit**
|
||||
|
||||
Run focused tests and build. Commit:
|
||||
|
||||
```bash
|
||||
git commit -m "feat: add ebook advanced reader settings"
|
||||
```
|
||||
|
||||
## Self-Review
|
||||
|
||||
- The plan covers server persistence first, matching the user's explicit preference.
|
||||
- The plan keeps ebook features in core and metadata in plugins, matching the audiobook pattern.
|
||||
- The plan avoids extra ebook catalog tables; added tables are reader-user-state only.
|
||||
- The biggest implementation risk is annotation rendering against Foliate selection APIs; Task 2 isolates it after backend persistence exists.
|
||||
- The migration filename is intentionally after existing ebook migrations and can be adjusted if another migration is added before implementation.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Ebook Reader Shell Parity Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Bring the core ebook reader up to the first useful parity slice from `personal/silo-plugin-ebooks`: TOC, search, progress scrub, reader settings, and keyboard navigation.
|
||||
|
||||
**Architecture:** Keep the current core reader API and persistence model. Extend `FoliateBookReader` with imperative methods for TOC, search, href/fraction navigation, and runtime style updates; keep `EbookReader` responsible for the chrome/panels/settings state. Store reader preferences in localStorage for this first local slice so no backend schema is added.
|
||||
|
||||
**Tech Stack:** React 19, React Router, TanStack Query, foliate-js custom element, existing Silo UI components, Vitest.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Reader Handle Capabilities
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/reader/FoliateBookReader.tsx`
|
||||
- Test: `web/src/reader/FoliateBookReader.test.ts`
|
||||
|
||||
- [ ] Add exported types `ReaderSettings`, `ReaderSearchOptions`, `ReaderSearchResult`, and `ReaderReadyState`.
|
||||
- [ ] Extend `FoliateBookReaderHandle` with `goToFraction`, `goTo`, `search`, `clearSearch`, and `applySettings`.
|
||||
- [ ] Add a `settings?: ReaderSettings` prop and convert `readerStyles()` to accept settings.
|
||||
- [ ] Preserve existing progress save/restore behavior.
|
||||
- [ ] Verify with `pnpm vitest run src/reader/FoliateBookReader.test.ts`.
|
||||
|
||||
### Task 2: TOC and Search Panel
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
- Test: `web/src/pages/EbookReader.test.tsx`
|
||||
|
||||
- [ ] Add panel state for `toc`, `searchResults`, `searchTerm`, and current tab.
|
||||
- [ ] Render side panel tabs for Contents, Search, and Settings.
|
||||
- [ ] Wire TOC items to `readerRef.current.goTo(href)`.
|
||||
- [ ] Wire search form to `readerRef.current.search(term)` and result clicks to `goTo(cfi)`.
|
||||
- [ ] Verify with `pnpm vitest run src/pages/EbookReader.test.tsx`.
|
||||
|
||||
### Task 3: Reader Settings
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
- Modify: `web/src/reader/FoliateBookReader.tsx`
|
||||
- Test: `web/src/pages/EbookReader.test.tsx`
|
||||
|
||||
- [ ] Add localStorage-backed settings for theme, font size, font family, line height, margin, max width, spread, and flow.
|
||||
- [ ] Render compact controls in the Settings tab.
|
||||
- [ ] Apply settings live through `FoliateBookReader`.
|
||||
- [ ] Verify settings persist across remount in tests.
|
||||
|
||||
### Task 4: Progress Scrub and Keyboard Navigation
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/pages/EbookReader.tsx`
|
||||
- Modify: `web/src/reader/FoliateBookReader.tsx`
|
||||
- Test: `web/src/pages/EbookReader.test.tsx`
|
||||
|
||||
- [ ] Add a progress range input in the header.
|
||||
- [ ] Call `goToFraction` on commit/change and update display from relocate events.
|
||||
- [ ] Add ArrowLeft/ArrowRight keyboard navigation on the reader page.
|
||||
- [ ] Verify with page-level tests.
|
||||
|
||||
### Task 5: Local Verification
|
||||
|
||||
**Files:**
|
||||
- No source changes.
|
||||
|
||||
- [ ] Run `cd web && pnpm run build`.
|
||||
- [ ] Run `GOWORK=off go test ./internal/api/handlers ./internal/scanner`.
|
||||
- [ ] Keep changes local; do not push.
|
||||
@@ -0,0 +1,518 @@
|
||||
# Encrypt integration credentials at rest (issue #45) — implementation plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Do not edit this plan file while implementing.
|
||||
|
||||
**Issue:** [#45 — Encrypt arr integration credentials at rest (Requests + Autoscan)](https://github.com/Silo-Server/silo-server/issues/45)
|
||||
|
||||
> **Revision note (post-review):** This plan was revised after a code review. Key corrections baked in:
|
||||
> the arr backfill now **resolves the legacy `server_settings` reference before encrypting** (otherwise
|
||||
> rows holding a setting-name like `requests.radarr.api_key` would be encrypted as the *name*, not the
|
||||
> credential); the sensitive-settings allowlist is **audited from the config loader's real inputs**
|
||||
> (incl. `redis.sentinel_password` and legacy `s3.operational_*` aliases) rather than copied from the
|
||||
> admin redaction map; AAD binds to **row identity** (`table:column:pk`); read paths **pass through
|
||||
> legacy plaintext** and hard-fail only on corrupt ciphertext; `settingsRepo` is **re-wrapped after the
|
||||
> max-connections pool recreation**; history-import session tokens are **included**; and
|
||||
> `plugin_runtime_configs.config_value` is an **explicit, documented gap** (see Out of scope).
|
||||
|
||||
**Goal:** Stop storing third-party integration credentials in plaintext at rest. Today Sonarr/Radarr
|
||||
API keys, S3 keys, and a dozen other secrets sit in the database as naked `text` — `request_integrations.api_key_ref`
|
||||
and `autoscan_connections.api_key_ref` are resolved through a plaintext `server_settings` key/value
|
||||
store, and both resolvers contain a **ref-vs-literal fallback**: when the secret lookup returns empty,
|
||||
the stored string is used verbatim as the API key. This plan introduces a real symmetric-encryption
|
||||
layer (AES-256-GCM, `SECRET_KEY`-derived via HKDF), encrypts every recoverable server-owned credential
|
||||
**inline** in its owning row, deletes the ambiguous resolver fallback, and backfills existing rows
|
||||
automatically on startup.
|
||||
|
||||
**Decisions (locked with the requester):**
|
||||
- **Master key:** a **required** `SECRET_KEY` environment variable. The server fatals at startup if it
|
||||
is missing. The key lives outside Postgres, so encrypted secrets survive a full DB compromise/dump.
|
||||
- **Scope:** **all silo-server-owned credential storage** (was framed "full sweep"). Every recoverable
|
||||
server-owned secret — arr keys, S3 keys, all sensitive `server_settings`, and the per-table token
|
||||
columns — is covered. Two categories are deliberately *not* in this PR because they are a different
|
||||
*shape* of problem, not because they are unimportant: equality-looked-up secrets (need **hashing**,
|
||||
not encryption) and plugin runtime config JSONB (cross-repo + manifest-driven). Both are called out in
|
||||
Out of scope with follow-ups.
|
||||
- **Data model (best practice):** **inline-encrypted**. Store the ciphertext directly in the owning
|
||||
column and decrypt on read. Drop the `server_settings` "ref" indirection and the `SecretResolver`
|
||||
literal fallback entirely — this kills the ambiguity at its root and matches the repo's one existing
|
||||
good example (`oauth_completion.token_ciphertext` lives inline in its own row).
|
||||
- **Rollout:** idempotent Go startup **backfill** encrypts any plaintext (no-envelope-prefix) value in
|
||||
place on first boot after deploy. No manual steps.
|
||||
|
||||
**Architecture:** A new `internal/secret` package owns the one cryptographic primitive everything reuses:
|
||||
AES-256-GCM with a random 12-byte nonce, `base64.RawURLEncoding`, and a versioned `enc:v1:` envelope
|
||||
prefix; the data key is HKDF-SHA256–derived from `SECRET_KEY` with a domain label. Each ciphertext is
|
||||
GCM-bound to its logical **row** via additional-authenticated-data (`"table:column:<pk>"`, or
|
||||
`"server_settings:<key>"`) so a DB-write attacker cannot transplant a credential blob into another row
|
||||
or column. Three wiring seams consume the cipher: (1) an `EncryptedSettingsRepo` **decorator** around
|
||||
`catalog.ServerSettingsRepo` that transparently encrypts/decrypts the sensitive `server_settings` keys,
|
||||
so `config.LoadFromDB` and every downstream consumer keep seeing plaintext; (2) the two `api_key_ref`
|
||||
columns become inline ciphertext and the entire `SecretResolver` indirection is deleted from
|
||||
`internal/requests` and `internal/autoscan`; (3) a handful of per-table repos gain a `*secret.Cipher`
|
||||
and encrypt-on-write / decrypt-on-read. A startup backfill sweeps all groups idempotently — with a
|
||||
**special resolve-then-encrypt path for the two arr columns** so legacy `server_settings` references are
|
||||
collapsed to the real credential before encryption.
|
||||
|
||||
**Out of scope (each a deliberate, documented decision — not an oversight):**
|
||||
- **Equality-looked-up secrets → need hashing, not encryption.** `api_keys.api_key`
|
||||
(`WHERE api_key = $1`) and `webhook_sync_connections.webhook_secret` (`WHERE webhook_secret = $1`,
|
||||
[repo.go:101](internal/webhooksync/repo.go:101)) are matched by exact value. AES-GCM uses a fresh
|
||||
random nonce, so the same plaintext encrypts differently each time — encrypting these would silently
|
||||
break API-key auth and inbound-webhook routing. The correct fix is a deterministic **hash** column +
|
||||
lookup-by-hash, a distinct lookup-semantics + migration change. **Follow-up issue.**
|
||||
- **`plugin_runtime_configs.config_value` → cross-repo + manifest-driven.** This is opaque plugin-defined
|
||||
JSONB ([runtime_config.go:55](internal/plugins/runtime_config.go:55)); some plugins store secrets in it
|
||||
(e.g. the legacy one-time copy of `introdb.api_key` at [main.go:2341](cmd/silo/main.go:2341)). It is
|
||||
read/written only through `RuntimeConfigStore` *in this repo*, so whole-blob encrypt-on-write /
|
||||
decrypt-on-read at that store is technically feasible — **but** host-side plugin runtime lives in the
|
||||
separate **`Silo`** repo (per CLAUDE.md) and may read this table directly, and which fields are secret
|
||||
is declared in each plugin's manifest. Doing this correctly needs a coordinated, manifest-aware design
|
||||
across repos. **Explicitly deferred — known gap.** Note: while deferred, the legacy `introdb.api_key`
|
||||
copy means that secret still exists in plaintext inside plugin config even after the `server_settings`
|
||||
copy is encrypted; the follow-up must re-encrypt or purge it.
|
||||
- **`plex_sync_connections`** — dead table, zero Go references (superseded by `webhook_sync_connections`
|
||||
in migration `060`). Not touched.
|
||||
- **Already-protected / non-secret:** `users.password_hash` (bcrypt), `oauth_completion.token_ciphertext`
|
||||
(already AES-GCM), `abs_sessions.token_hash` / `oauth_completion.code_hash` (hashed), `oauth_session.state`
|
||||
(equality CSRF), `subtitle_provider_config.username` (a username, not a secret),
|
||||
`history_import_plex_sessions.pin_id`/`pin_code` (transient OAuth pairing handshake, not a stored
|
||||
access credential).
|
||||
- `internal/auth/oauth_store.go` is left as-is (its key is jwt-derived and its data is ephemeral).
|
||||
Optionally a shared low-level seal/open helper can be extracted later; not required here.
|
||||
|
||||
**Tech Stack:** Go 1.26.3 (module declares `go 1.26.3`; stdlib `crypto/hkdf` is available — no new
|
||||
dependency). PostgreSQL via `pgxpool`. **No schema migration** — every target column is already `text`
|
||||
(or `jsonb` left untouched) and now simply holds ciphertext; the backfill is pure Go. No frontend or
|
||||
client (silo-android / silo-apple) changes — API responses already mask these values to `has_api_key` /
|
||||
`has_credentials` booleans, so no contract changes.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Running tests
|
||||
|
||||
The `internal/secret` package is pure Go (no libvips/CGO). If a Go toolchain is on the host:
|
||||
|
||||
```bash
|
||||
GOWORK=off go test ./internal/secret/... -v
|
||||
```
|
||||
|
||||
`GOWORK=off` is required when running from a `.claude/worktrees/` checkout — the parent `go.work`
|
||||
otherwise pins the module to `main` and the build fails with "main module does not contain package".
|
||||
|
||||
If the host has no Go toolchain, run in a throwaway container (a named volume caches module downloads):
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/secret/... -v
|
||||
```
|
||||
|
||||
Full backend build/vet before opening the MR: `GOWORK=off go build ./...` then `make lint` and
|
||||
`make verify-local-paths`.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
**New files:**
|
||||
- `internal/secret/cipher.go` — **create**. `Cipher` type; `New`, `Encrypt`, `Decrypt`, `IsEncrypted`,
|
||||
`EncryptIfPlaintext`; envelope + HKDF + version dispatch. Row-bound AAD passed by callers.
|
||||
- `internal/secret/cipher_test.go` — **create**. Round-trip, tamper/AAD-mismatch rejection, wrong-key,
|
||||
`IsEncrypted` edge cases, version dispatch, idempotency.
|
||||
- `internal/secret/backfill.go` — **create**. Generic in-place backfill for literal-credential columns +
|
||||
the special resolve-then-encrypt backfill for the two arr columns.
|
||||
- `internal/secret/backfill_test.go` — **create**. Idempotency, mixed plaintext/ciphertext rows, empty
|
||||
values skipped, failure isolation, arr resolve-then-encrypt.
|
||||
|
||||
**Modified files:**
|
||||
- `internal/config/bootstrap.go` — add `SecretKey []byte` to `BootstrapConfig`; read `SECRET_KEY`; fatal
|
||||
if missing or `< 32` chars.
|
||||
- `cmd/silo/main.go` — construct `*secret.Cipher` after bootstrap; build the `EncryptedSettingsRepo`;
|
||||
run the settings + column backfills after migrations and before settings are read; **re-wrap
|
||||
`settingsRepo` after the max-connections pool recreation (line ~399)**; thread the cipher into the
|
||||
affected repo constructors; delete the `SetSecretResolver` wiring.
|
||||
- `internal/catalog/encrypted_settings_repo.go` — **create**. `EncryptedSettingsRepo` decorator +
|
||||
exported, **audited** `SensitiveSettingKeys` (single source of truth) + `BackfillSensitiveSettings`.
|
||||
- `internal/catalog/server_settings_repo.go` — unchanged behavior; remains the raw inner store.
|
||||
- `internal/api/handlers/admin.go` — replace the local `sensitiveSettingKeys` map with
|
||||
`catalog.SensitiveSettingKeys` (redaction now shares the audited allowlist; closes the prior gap where
|
||||
`redis.sentinel_password` and `s3.operational_*` were returned in plaintext).
|
||||
- `internal/requests/repository.go` — `Repository` gains `*secret.Cipher`; encrypt `api_key_ref` on
|
||||
insert/update (AAD bound to integration id); decrypt (pass-through legacy plaintext) in `scanIntegration`.
|
||||
- `internal/requests/service.go` — delete `SecretResolver` interface, `secrets` field,
|
||||
`SetSecretResolver`, and `resolveAPIKey`; call sites use the already-decrypted `Integration.APIKeyRef`.
|
||||
- `internal/autoscan/repository.go` — `Repository` gains `*secret.Cipher`; encrypt `api_key_ref` on
|
||||
create/update (AAD bound to connection id); decrypt (pass-through legacy) in `scanConnection`.
|
||||
- `internal/autoscan/connection.go` — delete `SecretResolver` interface, `secrets` field, and the
|
||||
secret-resolution branch in `Resolve`; linked-integration path uses the requests repo's decrypted key.
|
||||
- `internal/api/autoscan_wiring.go` — drop the `AutoscanSecretResolver` param from `BuildAutoscanService`.
|
||||
- `internal/api/router.go` — drop the `SetSecretResolver(settingsRepo)` calls; pass the
|
||||
`EncryptedSettingsRepo` where `settingsRepo` is used; thread the cipher into repo constructors.
|
||||
- `internal/subtitles/pgrepo.go` — `*secret.Cipher`; encrypt `api_key` + `password` on upsert; decrypt on
|
||||
read. PK for AAD = `provider_name`.
|
||||
- `internal/watchsync/repository.go` — `*secret.Cipher`; encrypt `access_token` + `refresh_token` on
|
||||
upsert; decrypt in a `decryptConnection` helper called after every scan. PK for AAD = connection `id`.
|
||||
- `internal/webhooksync/repo.go` — `*secret.Cipher`; encrypt `access_token` on create; decrypt on scan.
|
||||
**`webhook_secret` is NOT encrypted** (equality lookup — carved out). PK for AAD = connection `id`.
|
||||
- `internal/historyimport/repo_admin.go` — `*secret.Cipher`; encrypt in `SetSourceAdminToken`; decrypt in
|
||||
`GetSourceWithAdminToken`. PK for AAD = source `id`.
|
||||
- `internal/historyimport/repo.go` — `*secret.Cipher`; encrypt `connect_access_token`
|
||||
(`history_import_connect_sessions`) + `auth_token` (`history_import_plex_sessions`) on insert/update;
|
||||
decrypt on scan. PK for AAD = session `id`.
|
||||
- `docs/architecture/secret-encryption.md` — **create**. Operator runbook (SECRET_KEY generation, backup,
|
||||
key-loss, rollback/downgrade).
|
||||
|
||||
**No migration files.** All target columns are already `text`; the backfill is Go-side.
|
||||
|
||||
---
|
||||
|
||||
## Reference tables
|
||||
|
||||
### What is encrypted vs hashed vs excluded
|
||||
|
||||
AAD column shows the GCM additional-authenticated-data string; `<pk>` is the row's stable primary key.
|
||||
|
||||
| Group | Table.column | AAD binding | Action |
|
||||
| --- | --- | --- | --- |
|
||||
| Arr keys | `request_integrations.api_key_ref` | `request_integrations:api_key_ref:<id>` | **Resolve-then-encrypt** (backfill) / encrypt inline (writes) |
|
||||
| Arr keys | `autoscan_connections.api_key_ref` | `autoscan_connections:api_key_ref:<id>` | **Resolve-then-encrypt** (backfill) / encrypt inline (writes) |
|
||||
| Settings | every key in `SensitiveSettingKeys` (see audited list below) | `server_settings:<key>` | **Encrypt via decorator** |
|
||||
| Subtitles | `subtitle_provider_config.api_key`, `.password` | `subtitle_provider_config:<col>:<provider_name>` | **Encrypt inline** |
|
||||
| Watch sync | `watch_provider_connections.access_token`, `.refresh_token` | `watch_provider_connections:<col>:<id>` | **Encrypt inline** |
|
||||
| Webhook sync | `webhook_sync_connections.access_token` | `webhook_sync_connections:access_token:<id>` | **Encrypt inline** |
|
||||
| History import | `history_import_sources.admin_token` | `history_import_sources:admin_token:<id>` | **Encrypt inline** |
|
||||
| History import | `history_import_connect_sessions.connect_access_token` | `history_import_connect_sessions:connect_access_token:<id>` | **Encrypt inline** |
|
||||
| History import | `history_import_plex_sessions.auth_token` | `history_import_plex_sessions:auth_token:<id>` | **Encrypt inline** |
|
||||
| — | `api_keys.api_key` | — | **Carved out → hash** (follow-up) |
|
||||
| — | `webhook_sync_connections.webhook_secret` | — | **Carved out → hash** (follow-up) |
|
||||
| — | `plugin_runtime_configs.config_value` | — | **Deferred → cross-repo/manifest design** (follow-up) |
|
||||
| — | `plex_sync_connections.*` | — | **Excluded** (dead table) |
|
||||
|
||||
### Audited `SensitiveSettingKeys` (single source of truth)
|
||||
|
||||
Built from the config loader's actual secret inputs ([internal/config/db_loader.go](internal/config/db_loader.go))
|
||||
plus the admin redaction map plus other known consumers — **not** just the current redaction map. The
|
||||
union is intentional and also fixes a pre-existing redaction leak. Implementers must re-audit
|
||||
`db_loader.go` and `grep` for any `*secret*`/`*token*`/`*password*`/`*api_key*` settings keys before
|
||||
finalizing.
|
||||
|
||||
```
|
||||
auth.jwt_secret
|
||||
s3.public_access_key s3.public_secret_key s3.public_token_secret
|
||||
s3.private_access_key s3.private_secret_key
|
||||
s3.user_db_access_key s3.user_db_secret_key
|
||||
s3.operational_access_key s3.operational_secret_key s3.operational_token_secret # legacy aliases still read as fallbacks (db_loader L165-192); migration 086 copied but did not delete them
|
||||
redis.url # may embed credentials (redis://:pass@host)
|
||||
redis.sentinel_password # read at db_loader L334 — MISSING from the old redaction map
|
||||
tmdb.api_key
|
||||
mdblist.api_key
|
||||
introdb.api_key # read in main.go (legacy → plugin copy)
|
||||
subtitle_ai.api_key
|
||||
recommendations.embedding_auth_token recommendations.openai_api_key # legacy alias (db_loader L409)
|
||||
watchsync.trakt.client_id watchsync.trakt.client_secret
|
||||
watchsync.simkl.client_id watchsync.simkl.client_secret
|
||||
requests.radarr.api_key requests.sonarr.api_key # legacy single-instance arr keys
|
||||
```
|
||||
|
||||
`auth.jwt_secret` IS encrypted at rest under `SECRET_KEY` (it is no longer the encryption root — `SECRET_KEY`
|
||||
is). `redis.url`/client-id entries are not high-value secrets but are kept in the set as a harmless safe
|
||||
superset and to match existing redaction.
|
||||
|
||||
### Envelope format
|
||||
|
||||
```
|
||||
enc:v1:<base64.RawURLEncoding( nonce[12] ‖ gcm.Seal(...) )>
|
||||
```
|
||||
|
||||
- `enc:v1:` — exactly 7 ASCII chars; not valid base64url, not a UUID/JWT/URL/integer prefix. Collision-proof
|
||||
against any real plaintext credential in practice.
|
||||
- `IsEncrypted(s)` ≡ `strings.HasPrefix(s, "enc:v1:")`.
|
||||
- Decrypt parses the version (`strings.SplitN(s, ":", 3)`), dispatches to the v1 opener; an unknown
|
||||
version returns an explicit `ErrUnknownVersion`. This is the forward-compat hook for a future `enc:v2:`
|
||||
key rotation — implement the dispatch now even though only `v1` exists.
|
||||
|
||||
### Read-path contract (applies to every decrypt site — decorator and repos)
|
||||
|
||||
1. `value == ""` → return `""`.
|
||||
2. `!IsEncrypted(value)` → **return the value unchanged** (legacy plaintext pass-through during the
|
||||
backfill window; the value is no worse than today). This is unambiguous — there is no longer any
|
||||
"ref vs literal" question because the `_ref` indirection is gone.
|
||||
3. `IsEncrypted(value)` → `Decrypt`; **any error (wrong key, tamper, truncation) is propagated, never
|
||||
swallowed and never returned as the ciphertext string.** A corrupt ciphertext fails the operation.
|
||||
|
||||
This reconciles the best-effort backfill (a skipped/failed row stays readable as plaintext) with the
|
||||
hard guarantee that a real `enc:v1:` value never silently degrades to using ciphertext as a credential.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: The `internal/secret` cipher package
|
||||
|
||||
**Files:** Create `internal/secret/cipher.go`, `internal/secret/cipher_test.go`.
|
||||
|
||||
This is the foundation; it has no dependencies on `catalog`/`config` and is independently testable.
|
||||
|
||||
- [ ] **Step 1: `Cipher` type + `New`.** Derive a 32-byte AES key with stdlib
|
||||
`hkdf.Key(sha256.New, masterKey, nil /*salt*/, "silo/data-encryption/v1", 32)`. Return an error if
|
||||
`len(masterKey) < 32`. Store the derived key as `[32]byte`. The type is safe for concurrent use.
|
||||
- [ ] **Step 2: `Encrypt(plaintext, aad string) (string, error)`.** Empty plaintext returns `"", nil`
|
||||
(do not produce an `enc:v1:` blob for an empty secret). Otherwise: `aes.NewCipher` → `cipher.NewGCM` →
|
||||
12-byte `rand` nonce → `gcm.Seal(nonce, nonce, []byte(plaintext), []byte(aad))` →
|
||||
`envelopePrefix + base64.RawURLEncoding.EncodeToString(sealed)`. `aad` is the row-bound context string.
|
||||
- [ ] **Step 3: `Decrypt(ciphertext, aad string) (string, error)`.** Strip + validate the version prefix;
|
||||
unknown version → `ErrUnknownVersion`. base64-decode; reject if shorter than the nonce size; split
|
||||
nonce/body; `gcm.Open(nil, nonce, body, []byte(aad))`. **Any failure returns an error — never fall back
|
||||
to returning the ciphertext or an empty string.** (Pass-through of *non-prefixed* plaintext is the
|
||||
caller's job per the read-path contract, not `Decrypt`'s.)
|
||||
- [ ] **Step 4: `IsEncrypted(s string) bool`** and `EncryptIfPlaintext(s, aad string) (string, bool, error)`.
|
||||
`EncryptIfPlaintext` returns `(s, false, nil)` when already encrypted or empty; otherwise encrypts and
|
||||
returns `(ct, true, nil)`. This is the idempotent primitive the backfill uses.
|
||||
- [ ] **Step 5: Tests** (`cipher_test.go`):
|
||||
- round-trip with AAD recovers plaintext; ciphertext has the `enc:v1:` prefix.
|
||||
- two encryptions of the same plaintext+AAD differ (random nonce works).
|
||||
- tamper: flip one byte of the decoded body → `Decrypt` errors.
|
||||
- AAD mismatch: encrypt with `"t:c:1"`, decrypt with `"t:c:2"` → errors (proves row binding).
|
||||
- wrong key: encrypt with key A, decrypt with key B → errors.
|
||||
- `New` with a `< 32`-byte key → error.
|
||||
- `IsEncrypted`: `"enc:v1:x"`→true, `"sa_abc"`→false, `""`→false, `"ENC:V1:x"`→false.
|
||||
- unknown version `"enc:v99:..."` → `ErrUnknownVersion`.
|
||||
- `EncryptIfPlaintext` is idempotent (second call is a no-op, `changed=false`).
|
||||
- [ ] **Step 6:** `GOWORK=off go test ./internal/secret/... -v` passes.
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `SECRET_KEY` bootstrap + cipher construction
|
||||
|
||||
**Files:** Modify `internal/config/bootstrap.go`, `cmd/silo/main.go`.
|
||||
|
||||
- [ ] **Step 1:** Add `SecretKey []byte` to `BootstrapConfig`. In `LoadBootstrap`, after the
|
||||
`DATABASE_URL` check, read `os.Getenv("SECRET_KEY")`; return an error (which the existing
|
||||
`log.Fatalf` path surfaces) if it is empty or `< 32` characters. Message must be actionable, e.g.
|
||||
`SECRET_KEY is required (>=32 chars); generate one with: openssl rand -base64 48`. `godotenv` already
|
||||
loads `.env`, so dev sets it there.
|
||||
- [ ] **Step 2:** In `cmd/silo/main.go`, immediately after `LoadBootstrap` returns, construct the cipher:
|
||||
`dataCipher, err := secret.New(bc.SecretKey)` and `log.Fatalf` on error. The cipher is threaded
|
||||
explicitly as a dependency — **never** a package-level global.
|
||||
- [ ] **Step 3:** Add `SECRET_KEY=...` to `.env.example` (or the documented dev env) and note it in the
|
||||
operator runbook (Task 7).
|
||||
- [ ] **Step 4:** `GOWORK=off go build ./cmd/silo` compiles; starting without `SECRET_KEY` fatals with the
|
||||
actionable message.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `server_settings` encryption decorator
|
||||
|
||||
**Files:** Create `internal/catalog/encrypted_settings_repo.go`; modify `internal/api/handlers/admin.go`,
|
||||
`cmd/silo/main.go`, `internal/api/router.go`.
|
||||
|
||||
- [ ] **Step 1: Build the audited allowlist.** Create exported `var SensitiveSettingKeys map[string]bool`
|
||||
in `internal/catalog` populated from the "Audited `SensitiveSettingKeys`" list above. **Do not** simply
|
||||
move the admin redaction map — re-audit `internal/config/db_loader.go` (and other `server_settings`
|
||||
consumers) for every secret-bearing key, and include the legacy aliases (`s3.operational_*`,
|
||||
`recommendations.openai_api_key`) and `redis.sentinel_password`. Update `admin.go`
|
||||
(`HandleGetSettings` redaction, `HandleGetSensitiveStatus`, the `PUT` response branch) to reference
|
||||
`catalog.SensitiveSettingKeys`. One allowlist now drives **both** redaction and encryption.
|
||||
- [ ] **Step 2:** Implement `EncryptedSettingsRepo` wrapping `*ServerSettingsRepo` + `*secret.Cipher`:
|
||||
- `Set(key, value)`: if `SensitiveSettingKeys[key]` and `value != ""`, encrypt with AAD
|
||||
`"server_settings:"+key` before delegating; else delegate raw.
|
||||
- `Get(key)` / `GetAll()`: delegate, then apply the **read-path contract** to each sensitive key
|
||||
(pass through legacy plaintext; decrypt prefixed values; propagate decrypt errors).
|
||||
- [ ] **Step 3:** Add `BackfillSensitiveSettings(ctx)` on the decorator: for each key in
|
||||
`SensitiveSettingKeys`, read the raw inner value; if non-empty and not `IsEncrypted`, `inner.Set` the
|
||||
encrypted value (AAD `"server_settings:"+key`). Idempotent. Returns a count + aggregated errors.
|
||||
- [ ] **Step 4:** In `cmd/silo/main.go`, build `settingsRepo` as the `EncryptedSettingsRepo` wrapping the
|
||||
raw `catalog.NewServerSettingsRepo(pool)`. Confirm the auto-generate-`jwt_secret` step still works: the
|
||||
`GetAll` map carries plaintext, the `== ""` check is unchanged, and `Set("auth.jwt_secret", encoded)`
|
||||
now writes **encrypted** while the in-memory map keeps the plaintext for `LoadFromDB`. Any function
|
||||
currently typed to the concrete `*catalog.ServerSettingsRepo` changes to the `ServerSettingsStore`
|
||||
interface that both already satisfy.
|
||||
- [ ] **Step 5 (re-wrap after pool recreation):** `cmd/silo/main.go` rebuilds a **raw**
|
||||
`catalog.NewServerSettingsRepo(pool)` after recreating the pool for `max_connections` (line ~399).
|
||||
Re-wrap it there too (`settingsRepo = catalog.NewEncryptedSettingsRepo(catalog.NewServerSettingsRepo(pool), dataCipher)`)
|
||||
so every later consumer gets the encrypting repo, not a raw one. Audit for any other raw construction
|
||||
of the settings repo and wrap each.
|
||||
- [ ] **Step 6:** Tests: decorator round-trip (raw store holds `enc:v1:`, decorator `Get` returns
|
||||
plaintext); non-sensitive keys pass through unencrypted; legacy-plaintext pass-through on read;
|
||||
`BackfillSensitiveSettings` idempotency.
|
||||
- [ ] **Step 7:** `GOWORK=off go build ./...` compiles; admin settings GET still redacts (now including
|
||||
`redis.sentinel_password`); a sensitive key written via the admin API lands as `enc:v1:` in the DB.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Inline-encrypt arr keys + delete the `SecretResolver` indirection
|
||||
|
||||
**Files:** Modify `internal/requests/repository.go`, `internal/requests/service.go`,
|
||||
`internal/autoscan/repository.go`, `internal/autoscan/connection.go`,
|
||||
`internal/api/autoscan_wiring.go`, `internal/api/router.go`, `cmd/silo/main.go`.
|
||||
|
||||
> The legacy-data resolution for these two columns is handled by the **resolve-then-encrypt backfill** in
|
||||
> Task 6 (a row may currently hold a `server_settings` key name, not a literal key). The repo write path
|
||||
> below only ever encrypts the literal credential an admin submits going forward.
|
||||
|
||||
- [ ] **Step 1:** `requests.NewRepository` gains a `*secret.Cipher` parameter. In `insertIntegration` and
|
||||
`updateIntegration`, encrypt `i.APIKeyRef` (AAD `"request_integrations:api_key_ref:"+i.ID`) before
|
||||
binding — preserve the existing `CASE WHEN $5 = '' THEN api_key_ref ELSE $5 END` keep-existing
|
||||
semantics (encrypt only the non-empty incoming value). In `scanIntegration`, apply the read-path
|
||||
contract to the scanned `api_key_ref`.
|
||||
- [ ] **Step 2:** Delete `requests.SecretResolver` (interface), the `Service.secrets` field,
|
||||
`Service.SetSecretResolver`, and `Service.resolveAPIKey`. Every former `resolveAPIKey(ctx, integration)`
|
||||
call site uses `strings.TrimSpace(integration.APIKeyRef)` directly (the repo already decrypted it).
|
||||
- [ ] **Step 3:** `autoscan.NewRepository` gains a `*secret.Cipher`. Encrypt `api_key_ref` in
|
||||
`CreateConnection` / `UpdateConnection` (AAD `"autoscan_connections:api_key_ref:"+id`, same
|
||||
keep-existing CASE semantics); apply the read-path contract in `scanConnection`.
|
||||
- [ ] **Step 4:** In `internal/autoscan/connection.go`, delete the `SecretResolver` interface, the
|
||||
`ConnectionResolver.secrets` field, and the secret-resolution branch in `Resolve`. For a linked Requests
|
||||
integration, `RequestIntegrationLookup.Get` returns the requests repo's **already-decrypted** key — use
|
||||
it directly. For a standalone connection, the autoscan repo already decrypted it. `Resolve` now only
|
||||
resolves base URL + linkage and returns plaintext.
|
||||
- [ ] **Step 5:** Remove the `AutoscanSecretResolver` parameter from `BuildAutoscanService`
|
||||
(`internal/api/autoscan_wiring.go`) and drop the `SetSecretResolver(settingsRepo)` calls in
|
||||
`internal/api/router.go` and `cmd/silo/main.go` (including the reconcile-worker service). Thread
|
||||
`dataCipher` into the requests/autoscan repo constructors at all wiring sites.
|
||||
- [ ] **Step 6:** Verify no API response serializes the raw key: the integration/connection response DTOs
|
||||
still emit only `has_api_key`. Add/extend a test asserting the create→store→read round-trip yields the
|
||||
original key, and that the stored column value is `enc:v1:`-prefixed.
|
||||
- [ ] **Step 7:** `GOWORK=off go build ./...`; `GOWORK=off go test ./internal/requests/... ./internal/autoscan/...`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Per-table credential columns
|
||||
|
||||
**Files:** Modify `internal/subtitles/pgrepo.go`, `internal/watchsync/repository.go`,
|
||||
`internal/webhooksync/repo.go`, `internal/historyimport/repo_admin.go`, `internal/historyimport/repo.go`,
|
||||
plus their constructor call sites.
|
||||
|
||||
Each repo gains a `*secret.Cipher` constructor arg and follows encrypt-on-write / decrypt-on-read with a
|
||||
row-bound AAD. Preserve existing keep-existing CASE/COALESCE semantics — encrypt only non-empty incoming
|
||||
values. All read paths use the read-path contract (pass through legacy plaintext, hard-fail corrupt
|
||||
ciphertext). These columns **never** had the resolver indirection — they always stored literal
|
||||
credentials — so no resolve-then-encrypt is needed (unlike the arr columns).
|
||||
|
||||
- [ ] **Step 1: subtitles** — encrypt `api_key` + `password` in `UpsertProviderConfig`; decrypt in
|
||||
`ListProviderConfigs` and `GetProviderConfig`. Keep the existing `HasAPIKey`/`HasCredentials` flags
|
||||
(set them from the decrypted values). AAD PK = `provider_name`.
|
||||
- [ ] **Step 2: watchsync** — encrypt `access_token` + `refresh_token` in `UpsertConnection`. Add a
|
||||
`decryptConnection(*Connection)` helper and call it after **every** scan path (list/get/by-provider/etc.
|
||||
— there are several). AAD PK = connection `id`.
|
||||
- [ ] **Step 3: webhooksync** — encrypt `access_token` in `CreateConnection`; decrypt in `scanConnection`.
|
||||
**Do NOT touch `webhook_secret`** — it is equality-looked-up in `GetConnectionBySecret`
|
||||
(`WHERE webhook_secret = $1`) and is carved out for the hashing follow-up. AAD PK = connection `id`.
|
||||
- [ ] **Step 4: historyimport admin token** — encrypt in `SetSourceAdminToken`; decrypt in
|
||||
`GetSourceWithAdminToken`; `ClearSourceAdminToken` (writes NULL) is unchanged. AAD PK = source `id`.
|
||||
- [ ] **Step 5: historyimport session tokens** — in `internal/historyimport/repo.go`, encrypt
|
||||
`connect_access_token` (`history_import_connect_sessions`) and `auth_token`
|
||||
(`history_import_plex_sessions`) on insert/update; decrypt on scan (note the existing
|
||||
`COALESCE(auth_token, '')` read). These are read by session `id`, not by token value. `pin_id`/`pin_code`
|
||||
are transient pairing artifacts — leave them. AAD PK = session `id`.
|
||||
- [ ] **Step 6:** Thread `dataCipher` into each constructor at its wiring site (`router.go` / `main.go`).
|
||||
- [ ] **Step 7:** `GOWORK=off go build ./...`; package tests for each modified repo pass.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Idempotent startup backfill
|
||||
|
||||
**Files:** Create `internal/secret/backfill.go`, `internal/secret/backfill_test.go`; modify `cmd/silo/main.go`.
|
||||
|
||||
- [ ] **Step 1: Generic in-place backfill** for the literal-credential columns (subtitle ×2, watchsync ×2,
|
||||
webhook `access_token`, historyimport `admin_token`, historyimport session `connect_access_token` +
|
||||
`auth_token`). Per `BackfillTarget {table, column, idColumn, aadPrefix}`: select rows where the column
|
||||
is non-null, non-empty, and `NOT LIKE 'enc:v1:%'`; for each, compute AAD = `aadPrefix + ":" + id`,
|
||||
`EncryptIfPlaintext`, and `UPDATE ... SET col = $1 WHERE id = $2 AND col = $3` (the `AND col = $3` guard
|
||||
makes concurrent multi-node boots converge without double-encrypting). Empty values are skipped (never
|
||||
encrypt `""`).
|
||||
- [ ] **Step 2: Resolve-then-encrypt backfill for the two arr columns** (`request_integrations.api_key_ref`,
|
||||
`autoscan_connections.api_key_ref`). This is **separate** from Step 1 because legacy rows may hold a
|
||||
`server_settings` key name rather than a literal key. For each row whose value is not already
|
||||
`enc:v1:`-prefixed, replicate the old `resolveAPIKey` semantics **once**:
|
||||
1. `resolved := encryptedSettingsRepo.Get(ctx, value)` (the decrypting decorator — handles both a
|
||||
sensitive-and-encrypted target like `requests.radarr.api_key` and a plaintext custom key).
|
||||
2. If `resolved` is non-empty, the row held a *reference* → encrypt `resolved` inline.
|
||||
Else the row held a *literal* key → encrypt `value` inline.
|
||||
3. `UPDATE ... SET api_key_ref = <enc> WHERE id = $2 AND api_key_ref = <original>`, AAD bound to the row id.
|
||||
Run this **after** `BackfillSensitiveSettings` so the referenced settings are consistent (the decorator
|
||||
decrypts regardless, but ordering keeps it deterministic). Optionally log/flag now-orphaned
|
||||
`requests.{radarr,sonarr}.api_key` settings rows for a later cleanup (do not delete in this PR).
|
||||
- [ ] **Step 3: Failure posture (decided):** best-effort. Collect per-row/per-target errors, log each at
|
||||
ERROR, and emit a single startup summary line (`secret backfill: encrypted N, failed M`). A failed
|
||||
encrypt leaves the pre-existing plaintext (no *new* exposure vs. today) and reads still succeed via the
|
||||
read-path pass-through, so it must **not** block boot — reliability over a hard stop. (Read-path decrypt
|
||||
failures on an `enc:v1:` value still hard-fail the individual operation.)
|
||||
- [ ] **Step 4: Ordering in `cmd/silo/main.go`** — gate on migration-running modes only (mirror the
|
||||
existing migration guard; proxy/transcode nodes skip the backfill). Sequence:
|
||||
`LoadBootstrap (read SECRET_KEY) → open pool → RunMigrations → construct dataCipher → wrap settingsRepo →
|
||||
BackfillSensitiveSettings → generic column backfill → arr resolve-then-encrypt backfill → GetAll(decrypts)
|
||||
→ auto-gen jwt_secret (writes encrypted) → [pool recreate? re-wrap settingsRepo] → LoadFromDB(plaintext) → validate`.
|
||||
The settings backfill + `GetAll` must precede `LoadFromDB` so config consumers read decrypted values.
|
||||
- [ ] **Step 5: Concurrency note** — the `AND col = $3` idempotency guard is sufficient for the common
|
||||
single-primary deploy. For multi-primary races, optionally wrap the backfill in a dedicated Postgres
|
||||
advisory lock with a **fresh** lock id (do **not** reuse the migration `legacyBootstrapLocker` id — that
|
||||
would extend the migration lock's held duration unpredictably).
|
||||
- [ ] **Step 6: Tests** (`backfill_test.go`, against a test DB): seed plaintext rows → backfill →
|
||||
all become `enc:v1:` and decrypt to the originals; second run is a no-op (no double-encrypt); mixed
|
||||
plaintext/ciphertext rows handled; empty values skipped; one failing row does not abort the rest;
|
||||
**arr resolve-then-encrypt:** a row holding `requests.radarr.api_key` (with that settings key present)
|
||||
ends up encrypting the *resolved* credential, and a row holding a literal key encrypts the literal.
|
||||
- [ ] **Step 7:** Boot against a DB pre-seeded with plaintext credentials **and** a legacy arr row that
|
||||
references a settings key; confirm the summary log, that every target column is `enc:v1:` afterward, that
|
||||
the arr row decrypts to the real Radarr key (not the setting name), and that a second boot reports
|
||||
`encrypted 0`.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Verification, docs, follow-up
|
||||
|
||||
- [ ] **Step 1: Regression guard tests:**
|
||||
- API key auth still works — `api_keys.api_key` is stored as the literal `sa_`-prefixed token and
|
||||
`GetByKey` (`WHERE api_key = $1`) still resolves (proves it was *not* routed through encryption).
|
||||
- `auth.jwt_secret` is consistent across two simulated boots (encrypted at rest, plaintext to
|
||||
`LoadFromDB`), and the stored `server_settings` row is `enc:v1:`-prefixed.
|
||||
- Inbound webhook routing still works (`webhook_secret` untouched).
|
||||
- A legacy arr integration that referenced `requests.sonarr.api_key` still fulfills after backfill.
|
||||
- [ ] **Step 2:** `GOWORK=off go build ./...`, `make lint`, `make verify-local-paths`,
|
||||
`cd web && pnpm run lint && pnpm run format:check` (frontend unaffected — confirm green).
|
||||
- [ ] **Step 3:** Write `docs/architecture/secret-encryption.md`: how `SECRET_KEY` is generated, that it
|
||||
must be backed up **separately** from DB dumps (treat like a CA private key), what happens on key loss
|
||||
(encrypted secrets unrecoverable → re-enter integrations), the rollback/downgrade hazard (an old binary
|
||||
reads `enc:v1:auth.jwt_secret` as a literal JWT secret → all sessions invalid; document that downgrade
|
||||
requires decrypting settings back to plaintext or clearing `auth.jwt_secret`), and the envelope/version
|
||||
scheme for future rotation.
|
||||
- [ ] **Step 4: File the follow-up issues:**
|
||||
1. **Hash equality-lookup secrets** (`api_keys.api_key`, `webhook_sync_connections.webhook_secret`) —
|
||||
store a deterministic hash, look up by hash.
|
||||
2. **Encrypt plugin runtime config** (`plugin_runtime_configs.config_value`) — coordinated with the
|
||||
`Silo` host repo and plugin-manifest secret-field metadata; must also re-encrypt/purge the legacy
|
||||
plaintext `introdb.api_key` copy. Note the dead `plex_sync_connections` table as a drop candidate.
|
||||
- [ ] **Step 5:** Update the issue #45 acceptance check: no plaintext arr keys at rest (Requests +
|
||||
Autoscan); `api_key_ref` is unambiguous (inline ciphertext, no literal fallback); existing rows
|
||||
encrypted by the backfill (arr rows resolved-then-encrypted); S3 + the broader server-owned credential
|
||||
set covered; remaining gaps (equality-lookup hashing, plugin JSONB) tracked as follow-ups.
|
||||
|
||||
---
|
||||
|
||||
## Safety checklist (the implementer must not skip)
|
||||
|
||||
1. `SECRET_KEY` is read in `LoadBootstrap` and **fatals** if absent/short — never defaults to a zero key
|
||||
or a hash of `DATABASE_URL`.
|
||||
2. The cipher is constructed once, right after bootstrap, before any `settingsRepo` call — threaded as a
|
||||
dependency, never a global.
|
||||
3. The settings backfill + decryption happen **before** `config.LoadFromDB` consumes the map.
|
||||
4. The auto-generated `auth.jwt_secret` is written **encrypted**, while the in-memory map keeps plaintext
|
||||
for `LoadFromDB`.
|
||||
5. **`settingsRepo` is re-wrapped as the `EncryptedSettingsRepo` after the max-connections pool recreation
|
||||
(main.go ~399)** and at every other construction site — no raw settings repo escapes into later wiring.
|
||||
6. The two **arr columns** use the **resolve-then-encrypt** backfill (not in-place) so legacy
|
||||
`server_settings` references become the real credential before encryption; the other columns use the
|
||||
plain in-place backfill.
|
||||
7. `SensitiveSettingKeys` is **audited from the config loader's real inputs** (incl.
|
||||
`redis.sentinel_password` and the legacy `s3.operational_*` / `recommendations.openai_api_key`
|
||||
aliases), not copied from the old redaction map.
|
||||
8. `api_keys.api_key` and `webhook_sync_connections.webhook_secret` are **never** routed through
|
||||
encryption — they are equality-looked-up and would break auth/routing (separate hashing task).
|
||||
9. Every decrypt site obeys the **read-path contract**: pass through non-prefixed plaintext; **error
|
||||
(never fall back)** on a corrupt `enc:v1:` value. Audit every old `if resolved == ""` site removed in Task 4.
|
||||
10. AAD binds to **row identity** (`table:column:<pk>`); PKs in scope (integration/connection ids, uuids,
|
||||
`provider_name`) are stable. server_settings binds to its key.
|
||||
11. The backfill is prefix-gated and idempotent (`AND col = $3` guard); proxy/transcode nodes skip it.
|
||||
12. `plex_sync_connections` is not a target (dead table); `plugin_runtime_configs.config_value` is a
|
||||
documented deferred gap, not silently ignored; `subtitle_provider_config.username` stays plaintext.
|
||||
@@ -0,0 +1,377 @@
|
||||
# AI Services Core, Metadata Translation & Whisper ASR — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Design:** `docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md` (read it first; this plan does not repeat the rationale).
|
||||
|
||||
**Goal:** Extract the subtitle-AI LLM plumbing into shared packages (`internal/ai/llm`, `internal/ai/translate`, `internal/ai/jobrunner`), add AI translation of overviews/taglines into the existing localization tables with field provenance, and add Whisper ASR subtitle generation (`transcribe`, `transcribe_translate`). Single PR.
|
||||
|
||||
**Architecture:** New `internal/ai/*` packages consumed by a refactored `internal/subtitles/ai` and a new `internal/metadata/translation` service. One new Goose migration (jobs table, provenance columns, library auto-translate flag). New `ai.*` settings keys with loader fallback to legacy `subtitle_ai.*` rows (encrypted values are GCM-bound to their key — never rename rows in SQL). Frontend: AI Services settings page, metadata-editor translate action, library toggle, player "generate from audio" mode.
|
||||
|
||||
**Tech stack:** Go (chi, pgx), PostgreSQL via Goose SQL migrations, React + TypeScript (TanStack Query), Vitest.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Ordering and verification
|
||||
|
||||
Tasks 1–4 are a pure refactor and must leave subtitle translation behaviorally unchanged — run `go build ./... && go test ./internal/...` after each. Tasks 5–11 are backend features; 12–15 frontend; 16 final verification.
|
||||
|
||||
Before opening the MR: `make lint`, `cd web && pnpm run lint && pnpm run format:check`, `make verify-local-paths`.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
**Create**
|
||||
- `internal/ai/llm/client.go`, `client_test.go`, `config.go` — shared OpenAI-compatible client (chat + transcription)
|
||||
- `internal/ai/llm/transcribe.go` — multipart `/v1/audio/transcriptions`, `verbose_json` types
|
||||
- `internal/ai/translate/translate.go`, `translate_test.go` — generic segment batch translator (package `aitranslate`)
|
||||
- `internal/ai/jobrunner/runner.go`, `runner_test.go` — dispatch/semaphore/heartbeat/reaper/cancel registry
|
||||
- `internal/metadata/translation/{job.go,pgrepo.go,service.go,prompt.go,service_test.go}`
|
||||
- `internal/subtitles/ai/transcriber.go`, `transcriber_test.go` — ASR pipeline
|
||||
- `internal/playback/audio_extract.go` — ffmpeg chunked audio extraction helper
|
||||
- `internal/api/handlers/metadata_ai.go`
|
||||
- `migrations/sql/<timestamp>_ai_metadata_translation_and_asr.sql` (via `make migrate-create`)
|
||||
- `web/src/pages/admin-settings/AIServicesSettings.tsx`
|
||||
|
||||
**Modify**
|
||||
- `internal/subtitles/ai/{client.go→deleted,translator.go,service.go,engine.go,job.go,config.go,pgrepo.go}`
|
||||
- `internal/config/{config.go,db_loader.go}`; `internal/catalog/encrypted_settings_repo.go`
|
||||
- `internal/catalog/{localization_repo.go,detail.go}`; `internal/models/media.go`
|
||||
- `internal/metadata/service.go` (provider provenance + auto-enqueue hook)
|
||||
- `internal/api/handlers/subtitle_ai.go`; router registration; `cmd/silo/main.go` wiring
|
||||
- `web/src/pages/admin-settings/SubtitlesSettings.tsx`, `web/src/components/EditMetadataDialog.tsx`, `web/src/player/components/SubtitleTranslateModal.tsx`, library settings form, `web/src/api/` types/client, admin settings nav/route registration
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `internal/ai/llm` — shared client
|
||||
|
||||
- [ ] **Step 1: Move the chat client.** Create `internal/ai/llm` (package `llm`). Move `Client`, `chat` (export as `Chat`), `chatMessage` (export as `Message`), retry helpers (`sleepCtx`, `rateLimitBackoff`, `truncate`) from `internal/subtitles/ai/client.go`. New `llm.Config`:
|
||||
|
||||
```go
|
||||
type Config struct {
|
||||
BaseURL string // chat endpoint, no trailing /v1
|
||||
APIKey string
|
||||
ChatModel string
|
||||
ASRBaseURL string // empty = BaseURL
|
||||
ASRAPIKey string // empty = APIKey
|
||||
ASRModel string
|
||||
MaxConcurrentJobs int
|
||||
}
|
||||
```
|
||||
|
||||
Log lines lose the "subtitle" wording (they are shared now).
|
||||
|
||||
- [ ] **Step 2: Add `Transcribe`.** In `transcribe.go`:
|
||||
|
||||
```go
|
||||
type TranscribeRequest struct {
|
||||
Filename string
|
||||
Audio io.Reader
|
||||
Language string // optional ISO-639-1 hint
|
||||
Timeout time.Duration // per-request; sized to chunk length by the caller
|
||||
}
|
||||
type TranscriptionSegment struct{ Start, End float64; Text string }
|
||||
type Transcription struct{ Language string; Segments []TranscriptionSegment }
|
||||
func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (*Transcription, error)
|
||||
```
|
||||
|
||||
Multipart fields: `file`, `model` (=`ASRModel`), `response_format=verbose_json`, `temperature=0`, optional `language`. Reuse the same 429/5xx/transport retry-with-backoff loop as `Chat` (extract a shared `doWithRetry` rather than duplicating it). Empty `segments` in the response is an error (`transcription returned no segments`), not a silent fallback.
|
||||
|
||||
- [ ] **Step 3: Tests** (`client_test.go`, httptest): chat retries on 429 (honors `Retry-After`), 5xx, 200-with-error-object, empty choices; transcribe happy path (multipart fields present, segments parsed), missing-segments error, ASR base-url/key override falling back to chat values.
|
||||
|
||||
- [ ] **Step 4: Commit.** `refactor(ai): extract shared OpenAI-compatible LLM client with transcription support`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `internal/ai/translate` — generic segment translator
|
||||
|
||||
- [ ] **Step 1: Move the batch logic.** Package `aitranslate`. Move `buildIndexedJSON`, `extractJSONObject`, and the batch loop from `internal/subtitles/ai/translator.go`, generalized:
|
||||
|
||||
```go
|
||||
type Segment struct{ ID, Text string }
|
||||
type Request struct {
|
||||
Segments []Segment
|
||||
SystemPrompt string // caller-supplied; domain-specific
|
||||
BatchSize int
|
||||
ContextNeighbors int // preceding source segments sent untranslated
|
||||
}
|
||||
type ChatFn func(ctx context.Context, system, user string) (string, error)
|
||||
func Translate(ctx context.Context, chat ChatFn, req Request,
|
||||
onBatch func(batch []Segment, done, total int)) ([]Segment, error)
|
||||
```
|
||||
|
||||
Wire protocol unchanged: 1-based indexed JSON per batch, same-keys response, malformed-response retries (`maxRetries = 2`), completeness check. `ChatFn` keeps the package free of an `llm` dependency and trivially testable.
|
||||
|
||||
- [ ] **Step 2: Tests:** batch splitting/boundaries, context neighbors, code-fence tolerance, omitted-key retry then failure, ctx cancellation between batches.
|
||||
|
||||
- [ ] **Step 3: Commit.** `refactor(ai): extract generic batched segment translator`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `internal/ai/jobrunner` — shared job lifecycle
|
||||
|
||||
- [ ] **Step 1: Extract the runner.** Move dispatch/heartbeat/reaper/cancel mechanics from `internal/subtitles/ai/service.go` (lines ~136–319) behind:
|
||||
|
||||
```go
|
||||
type Store interface {
|
||||
Heartbeat(ctx context.Context, id int64) error
|
||||
ResetStaleJobs(ctx context.Context, before time.Time, message string) (int64, error)
|
||||
MarkCancelled(ctx context.Context, id int64, message string) error
|
||||
}
|
||||
type Runner struct{ /* baseCtx, sem (shared), store, logger, cancels, wg */ }
|
||||
func New(appCtx context.Context, sem chan struct{}, store Store, logger *slog.Logger) *Runner
|
||||
func (r *Runner) Recover() // reap + background reaper loop
|
||||
func (r *Runner) Dispatch(id int64, run func(ctx context.Context)) // semaphore + heartbeat + cancel registry
|
||||
func (r *Runner) Cancel(id int64) bool // true if an in-flight goroutine was cancelled
|
||||
```
|
||||
|
||||
Heartbeat-while-queued behavior (the comment block in `dispatch`) must be preserved verbatim — it is load-bearing for multi-instance safety. Constants (30 s heartbeat, 2 min stale, 1 min reaper) move here.
|
||||
|
||||
- [ ] **Step 2: Shared semaphore.** The semaphore is **constructed by the caller** (`cmd/silo`) at size `ai.max_concurrent_jobs` and passed to every `Runner`, so subtitle + metadata + ASR jobs share one bound.
|
||||
|
||||
- [ ] **Step 3: Tests:** dispatch bounded by a size-1 shared semaphore across two runners; cancel of queued job marks cancelled without running; reaper resets a stale heartbeat row (fake store).
|
||||
|
||||
- [ ] **Step 4: Commit.** `refactor(ai): extract shared job lifecycle runner`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Refactor `internal/subtitles/ai` onto the shared core
|
||||
|
||||
- [ ] **Step 1:** Delete `client.go`; `LLMTranslator` becomes an adapter: cues → `aitranslate.Segment` (ID = 1-based index, Text = joined lines), subtitle system prompt stays here, `splitCueLines` maps back. `Service` keeps job semantics but delegates lifecycle to `jobrunner.Runner`; `pgrepo` satisfies `jobrunner.Store`.
|
||||
- [ ] **Step 2:** Update `cmd/silo/main.go` wiring (build `llm.Client` + shared semaphore once; pass into the subtitle service).
|
||||
- [ ] **Step 3:** `go build ./... && go test ./internal/...` — subtitle translation behavior unchanged (same prompts, same batching, same job rows).
|
||||
- [ ] **Step 4: Commit.** `refactor(subtitles): consume shared AI core`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Migration
|
||||
|
||||
- [ ] **Step 1:** `make migrate-create NAME=ai_metadata_translation_and_asr`, then fill the generated file:
|
||||
|
||||
```sql
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE public.metadata_translation_jobs (
|
||||
id bigserial PRIMARY KEY,
|
||||
target_kind text NOT NULL, -- 'item' | 'season' | 'episode'
|
||||
content_id text NOT NULL,
|
||||
include_children boolean NOT NULL DEFAULT true,
|
||||
source_language text NOT NULL DEFAULT '',
|
||||
target_language text NOT NULL,
|
||||
engine text NOT NULL DEFAULT 'openai',
|
||||
model text NOT NULL DEFAULT '',
|
||||
status text NOT NULL DEFAULT 'pending', -- pending|running|completed|failed|cancelled
|
||||
progress double precision NOT NULL DEFAULT 0,
|
||||
progress_message text NOT NULL DEFAULT '',
|
||||
fields_done integer NOT NULL DEFAULT 0,
|
||||
fields_total integer NOT NULL DEFAULT 0,
|
||||
force boolean NOT NULL DEFAULT false,
|
||||
error_message text NOT NULL DEFAULT '',
|
||||
idempotency_key text NOT NULL,
|
||||
requested_by integer,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
heartbeat_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE UNIQUE INDEX metadata_translation_jobs_active_idempotency_idx
|
||||
ON public.metadata_translation_jobs (idempotency_key)
|
||||
WHERE status IN ('pending', 'running');
|
||||
CREATE INDEX metadata_translation_jobs_content_idx
|
||||
ON public.metadata_translation_jobs (content_id, created_at DESC);
|
||||
CREATE INDEX metadata_translation_jobs_status_idx
|
||||
ON public.metadata_translation_jobs (status) WHERE status IN ('pending', 'running');
|
||||
|
||||
-- Field provenance: 'provider' | 'ai' | 'manual'. manual > provider > ai.
|
||||
ALTER TABLE media_item_localizations
|
||||
ADD COLUMN overview_source text NOT NULL DEFAULT 'provider'
|
||||
CHECK (overview_source IN ('provider', 'ai', 'manual')),
|
||||
ADD COLUMN tagline_source text NOT NULL DEFAULT 'provider'
|
||||
CHECK (tagline_source IN ('provider', 'ai', 'manual'));
|
||||
ALTER TABLE season_localizations
|
||||
ADD COLUMN overview_source text NOT NULL DEFAULT 'provider'
|
||||
CHECK (overview_source IN ('provider', 'ai', 'manual'));
|
||||
ALTER TABLE episode_localizations
|
||||
ADD COLUMN overview_source text NOT NULL DEFAULT 'provider'
|
||||
CHECK (overview_source IN ('provider', 'ai', 'manual'));
|
||||
|
||||
ALTER TABLE media_folders
|
||||
ADD COLUMN auto_translate_metadata boolean NOT NULL DEFAULT false;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE media_folders DROP COLUMN auto_translate_metadata;
|
||||
ALTER TABLE episode_localizations DROP COLUMN overview_source;
|
||||
ALTER TABLE season_localizations DROP COLUMN overview_source;
|
||||
ALTER TABLE media_item_localizations DROP COLUMN tagline_source, DROP COLUMN overview_source;
|
||||
DROP TABLE IF EXISTS public.metadata_translation_jobs;
|
||||
-- +goose StatementEnd
|
||||
```
|
||||
|
||||
- [ ] **Step 2:** `make migrate-up` against the local stack (`docker compose up -d postgres redis`), then `make migrate-status`.
|
||||
- [ ] **Step 3: Commit.** `feat(metadata): migration for translation jobs, localization provenance, library auto-translate`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Settings & config
|
||||
|
||||
- [ ] **Step 1: Config structs** (`internal/config/config.go`): add `AIConfig` (fields mirroring `llm.Config`); slim `SubtitleAIConfig` to `Enabled`, `TranscribeEnabled`, `BatchSize`, `ContextNeighbors`; add `MetadataAIConfig{Enabled bool}`.
|
||||
- [ ] **Step 2: Loader** (`internal/config/db_loader.go`): load `ai.*` with legacy fallback, following the `recommendations.embedding_auth_token` precedent at line ~409:
|
||||
|
||||
```go
|
||||
cfg.AI.BaseURL = stringOr(m, "ai.base_url", stringOr(m, "subtitle_ai.base_url", "https://api.openai.com"))
|
||||
cfg.AI.APIKey = stringOr(m, "ai.api_key", stringOr(m, "subtitle_ai.api_key", ""))
|
||||
cfg.AI.ChatModel = stringOr(m, "ai.chat_model", stringOr(m, "subtitle_ai.chat_model", "gpt-4o-mini"))
|
||||
// ai.max_concurrent_jobs ← subtitle_ai.max_concurrent_jobs ← 2
|
||||
// ai.asr_model (default "whisper-1"), ai.asr_base_url, ai.asr_api_key (default "")
|
||||
// subtitle_ai.transcribe_enabled, metadata_ai.enabled (default false)
|
||||
```
|
||||
|
||||
**Never** rename the legacy rows in SQL — encrypted values are GCM-bound to their setting key (`internal/catalog/encrypted_settings_repo.go`).
|
||||
|
||||
- [ ] **Step 3:** Add `ai.api_key`, `ai.asr_api_key` to `sensitiveSettingKeys`.
|
||||
- [ ] **Step 4:** Readiness helpers: subtitle translate = `SubtitleAI.Enabled && AI chat ready`; ASR = `SubtitleAI.TranscribeEnabled && AI.ASRModel != "" && (asr or chat base URL set)`; metadata = `MetadataAI.Enabled && AI chat ready`.
|
||||
- [ ] **Step 5: Commit.** `feat(config): shared ai.* settings with legacy subtitle_ai fallback`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Provenance-aware localization writes
|
||||
|
||||
- [ ] **Step 1: Models** (`internal/models/media.go`): add `OverviewSource`/`TaglineSource` to `MediaItemLocalization`, `OverviewSource` to `SeasonLocalization`/`EpisodeLocalization`.
|
||||
- [ ] **Step 2: Provider upsert rules** (`internal/catalog/localization_repo.go`): rewrite the three `Upsert` statements so, per AI-writable field:
|
||||
- existing source `manual` → keep existing value and source;
|
||||
- incoming value empty → keep existing value and source;
|
||||
- otherwise → take incoming value, set source `provider`.
|
||||
|
||||
Single-statement `ON CONFLICT … DO UPDATE` with `CASE` (no read-modify-write). Pattern for one field:
|
||||
|
||||
```sql
|
||||
overview = CASE
|
||||
WHEN media_item_localizations.overview_source = 'manual' THEN media_item_localizations.overview
|
||||
WHEN EXCLUDED.overview = '' THEN media_item_localizations.overview
|
||||
ELSE EXCLUDED.overview END,
|
||||
overview_source = CASE
|
||||
WHEN media_item_localizations.overview_source = 'manual' THEN media_item_localizations.overview_source
|
||||
WHEN EXCLUDED.overview = '' THEN media_item_localizations.overview_source
|
||||
ELSE 'provider' END
|
||||
```
|
||||
|
||||
- [ ] **Step 3: AI upsert methods**: `UpsertAITranslation(ctx, contentID, language string, overview, tagline *string, force bool)` per repo (nil pointer = field not part of this write). Insert path populates only the translated fields (other text columns empty). Update path per field: write when existing source is `ai`, existing value is empty, or `force` — never when `manual`; `force` may overwrite `provider` (the admin asked). Sets source `ai`.
|
||||
- [ ] **Step 4: Serving hardening** (`internal/catalog/detail.go`): verify `LocalizeItemModel` / `LocalizeSeasonModel` / `LocalizeEpisodeModel` only override base fields with **non-empty** localization values (AI rows carry empty titles). Fix any field that clobbers; add a regression test.
|
||||
- [ ] **Step 5: Tests** for the upsert matrix (provider-over-ai, ai-skips-provider-unless-force, manual-untouchable, empty-never-blanks), following the existing repo/service test patterns in `internal/catalog`.
|
||||
- [ ] **Step 6: Commit.** `feat(catalog): field provenance for localizations with provider/ai/manual precedence`
|
||||
|
||||
---
|
||||
|
||||
## Task 8: `internal/metadata/translation` service
|
||||
|
||||
- [ ] **Step 1: Job model + repo** (`job.go`, `pgrepo.go`): mirror `internal/subtitles/ai/{job.go,pgrepo.go}` shapes against `metadata_translation_jobs`; satisfy `jobrunner.Store`. Idempotency key = SHA-256 of `content_id|target_kind|target_language|model`.
|
||||
- [ ] **Step 2: Service** (`service.go`): `Enqueue`, `GetJob`, `ListJobs(contentID)`, `Cancel`, `Recover`, using `jobrunner.Runner` + the shared semaphore. The run loop:
|
||||
1. Expand targets: `item` → item overview+tagline; series item with `include_children` → + every season overview + every episode overview; `season` (+children → its episodes); `episode` → its overview. Source text comes from the **base** rows (default metadata language); `source_language` recorded from the item's `default_metadata_language`.
|
||||
2. Skip-if-filled: drop any field whose target-language localization value is already non-empty, unless `force`. All skipped → complete immediately ("Nothing to translate").
|
||||
3. Build `aitranslate.Segment`s (IDs `item:overview`, `item:tagline`, `season:<content_id>:overview`, `episode:<content_id>:overview`), translate with `prompt.go`'s system prompt (names the title/year, "translate media catalog descriptions", preserve proper nouns/character names/tone, no added information), `BatchSize` = package constant `metadataBatchSize = 10`, `ContextNeighbors = 0`.
|
||||
4. Per batch: provenance-aware AI upserts (Task 7) + `fields_done`/progress update. Persisting per batch means a cancelled job keeps completed fields.
|
||||
- [ ] **Step 3: Tests** (fake repos + fake `ChatFn`): series expansion counts, skip-if-filled short-circuit (zero chat calls), force overwrites `ai`+`provider` but never `manual`, per-batch persistence on cancellation mid-job.
|
||||
- [ ] **Step 4: Commit.** `feat(metadata): AI translation service for overviews and taglines`
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Ingestion hook + library flag
|
||||
|
||||
- [ ] **Step 1:** Switch the provider localization writes in `internal/metadata/service.go` (item ~line 1476, season ~2921, episode ~3080) to the provenance-aware upserts — no other behavior change.
|
||||
- [ ] **Step 2:** Thread `auto_translate_metadata` through the `MediaFolder` model, folder repo scan/update, and the folder settings API payload.
|
||||
- [ ] **Step 3: Auto-enqueue hook.** After an item's refresh persists (single point at the end of the item flow — not per season/episode), when: folder flag set, folder `metadata_language` non-empty and ≠ item `default_metadata_language`, metadata AI ready, and the item's target-language localization is missing overview (or tagline) → `Enqueue` a non-force `item` job with `include_children=true`. Fire-and-forget (log on error); the active-idempotency index plus skip-if-filled make repeat refreshes free.
|
||||
- [ ] **Step 4: Commit.** `feat(metadata): per-library auto-translate fallback on refresh`
|
||||
|
||||
---
|
||||
|
||||
## Task 10: Metadata AI API
|
||||
|
||||
- [ ] **Step 1:** `internal/api/handlers/metadata_ai.go`, mirroring `subtitle_ai.go` shapes:
|
||||
- `GET /api/v1/metadata/ai/status` → `{enabled}`
|
||||
- `POST /api/v1/metadata/ai/translate` `{content_id, target_kind, target_language, include_children, force}` → `{job}` (joins an in-flight duplicate, same as subtitles)
|
||||
- `GET /api/v1/metadata/ai/jobs/{job_id}`, `POST /api/v1/metadata/ai/jobs/{job_id}/cancel`, `GET /api/v1/metadata/ai/jobs?content_id=…`
|
||||
- [ ] **Step 2:** Gate with the same permission as metadata editing (metadata curation permission; see `internal/catalog/update.go` callers). Register routes; wire the service in `cmd/silo/main.go`.
|
||||
- [ ] **Step 3:** Handler tests following `internal/api/handlers` conventions (validation: bad kind, missing language, disabled engine → 400/503 mapping).
|
||||
- [ ] **Step 4: Commit.** `feat(api): metadata AI translation endpoints`
|
||||
|
||||
---
|
||||
|
||||
## Task 11: Whisper ASR
|
||||
|
||||
- [ ] **Step 1: Audio extraction** (`internal/playback/audio_extract.go`), alongside the existing subtitle extraction helpers:
|
||||
|
||||
```go
|
||||
// ExtractAudioChunks extracts one audio track to 16 kHz mono WAV chunks in dir.
|
||||
// Returns ordered chunk paths; chunkSeconds = 600 in production.
|
||||
func ExtractAudioChunks(ctx context.Context, filePath string, audioTrackIndex int,
|
||||
dir, ffmpegPath string, chunkSeconds int) ([]string, error)
|
||||
```
|
||||
|
||||
Single ffmpeg pass: `-vn -map 0:a:<idx> -ac 1 -ar 16000 -c:a pcm_s16le -f segment -segment_time <n> <dir>/chunk%05d.wav`.
|
||||
|
||||
- [ ] **Step 2: Transcriber** (`internal/subtitles/ai/transcriber.go`):
|
||||
|
||||
```go
|
||||
type Transcriber interface {
|
||||
Transcribe(ctx context.Context, req TranscribeJobRequest,
|
||||
onChunk func(cues []SubtitleCue, done, total int)) ([]SubtitleCue, string /*detected lang*/, error)
|
||||
}
|
||||
```
|
||||
|
||||
`WhisperTranscriber` implementation: temp dir (`os.MkdirTemp`, removed on all exit paths) → `ExtractAudioChunks` → per chunk `llm.Client.Transcribe` (per-request timeout ∝ chunk duration; pass the job's language hint) → offset segment times by `chunkIndex*chunkSeconds` → cues (one per segment, wrapped to ≤2 lines ×~42 chars via a `wrapCueText` helper; whitespace-only segments dropped). Chunk processing order starts at the chunk containing `StartPosition`, then wraps (mirrors `reorderFromPosition` semantics).
|
||||
|
||||
- [ ] **Step 3: Wire kinds into the service.** `Service.run` branches on `job.Kind`:
|
||||
- `translate`: existing path.
|
||||
- `transcribe`: resolve audio track (`source_index` = audio index, `-1` → default/first; reject files without audio), transcribe (progress 10–70 %, live cue streaming via the existing notifier callbacks), store SRT as provider `transcribed`, language = hint or detected, release name `"<Language> (AI transcribed)"`, notify `SubtitleReady`.
|
||||
- `transcribe_translate`: `transcribe`, store the transcript track, then run the existing `Translator` on the cues (70–95 %), store provider `translated`, `result_subtitle_id` = translated track.
|
||||
|
||||
Enqueue validation per kind (ASR requires `TranscribeEnabled`; idempotency model component = `asr_model` or `asr_model+chat_model` for the chained kind).
|
||||
|
||||
- [ ] **Step 4:** API surface: `kind` field (optional, default `translate`) on the existing enqueue request in `subtitle_ai.go`; `transcribe_enabled` on the status response.
|
||||
- [ ] **Step 5: Tests** (`transcriber_test.go`, fake client): timestamp stitching across chunks, playhead-first chunk ordering, cue wrapping, empty-segments chunk → job error, temp-dir cleanup on failure (assert via `t.TempDir` layout).
|
||||
- [ ] **Step 6: Commit.** `feat(subtitles): Whisper ASR transcribe and transcribe_translate jobs`
|
||||
|
||||
---
|
||||
|
||||
## Task 12: Frontend — AI Services settings page
|
||||
|
||||
- [ ] **Step 1:** `web/src/pages/admin-settings/AIServicesSettings.tsx`: connection card (`ai.base_url`, `ai.api_key` via the sensitive-key pattern already used in `SubtitlesSettings.tsx:264`, `ai.chat_model`, `ai.asr_model`, optional `ai.asr_base_url`/`ai.asr_api_key`, `ai.max_concurrent_jobs`) + features card (`subtitle_ai.enabled`, `subtitle_ai.transcribe_enabled`, `metadata_ai.enabled`, `subtitle_ai.batch_size`, `subtitle_ai.context_neighbors`). Reads show effective values (new key, falling back to the legacy key when unset — same fallback order as the loader); writes always target the new `ai.*` keys.
|
||||
- [ ] **Step 2:** Remove the AI card from `SubtitlesSettings.tsx` (leave a link/hint to the new page); register the page in the admin settings nav + route following the existing page registrations.
|
||||
- [ ] **Step 3: Commit.** `feat(web): AI services settings page`
|
||||
|
||||
---
|
||||
|
||||
## Task 13: Frontend — metadata translate action
|
||||
|
||||
- [ ] **Step 1:** API client functions + types for the Task 10 endpoints.
|
||||
- [ ] **Step 2:** `EditMetadataDialog.tsx`: "Translate with AI" action (visible when `metadata_ai` status is enabled): target language (default = library `metadata_language`), `include_children` (series only), `force` checkbox; enqueue, poll the job every ~1.5 s, show `progress_message`/`fields_done`, invalidate the item detail query on completion.
|
||||
- [ ] **Step 3: Commit.** `feat(web): translate descriptions from the metadata editor`
|
||||
|
||||
---
|
||||
|
||||
## Task 14: Frontend — library auto-translate toggle
|
||||
|
||||
- [ ] **Step 1:** Add the `auto_translate_metadata` switch to the library settings form next to the metadata-language field (helper text: "When metadata providers have no translation for this library's language, translate descriptions with AI"). Disabled state with hint when `metadata_ai` is off.
|
||||
- [ ] **Step 2: Commit.** `feat(web): library auto-translate toggle`
|
||||
|
||||
---
|
||||
|
||||
## Task 15: Frontend — player "Generate from audio"
|
||||
|
||||
- [ ] **Step 1:** `SubtitleTranslateModal.tsx`: when `transcribe_enabled`, add a source option "Generate from audio (AI)" listing the file's audio tracks; submit with `kind: "transcribe"` (or `transcribe_translate` when the chosen output language differs from the audio language). Surface it prominently when no text subtitle source exists (today's dead end for bitmap-only files).
|
||||
- [ ] **Step 2: Commit.** `feat(web): generate subtitles from audio in the player`
|
||||
|
||||
---
|
||||
|
||||
## Task 16: Verification
|
||||
|
||||
- [ ] `go build ./... && go test ./...`
|
||||
- [ ] `make lint`
|
||||
- [ ] `cd web && pnpm run lint && pnpm run format:check`
|
||||
- [ ] `make verify-local-paths`
|
||||
- [ ] Manual smoke against the local stack: enqueue a metadata translation for a series (verify localization rows + provenance values), re-run a refresh (verify provider values overwrite `ai` rows and `manual` simulation survives), run a `transcribe` job on a short file against a local Whisper-compatible server.
|
||||
- [ ] MR description: problem, approach, link to the design doc, risks (Whisper endpoint variance, chunk-boundary artifacts), AI-use disclosure, screenshots of the new settings page / editor action / player mode.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,809 @@
|
||||
# Core Notifications — Web UI Implementation Plan (Phase 2 of 3)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Notifications UI in the silo-server web app: unread badge (sidebar + mobile header bell with dropdown), live toasts from the events WebSocket, an inbox page, per-category preference toggles in settings, and an admin announcements page.
|
||||
|
||||
**Architecture:** A `notifications` query-hook module talks to the Phase-1 REST API; one live-wiring hook subscribes to the `notifications` WS channel via the existing `RealtimeEventsProvider` ref-counted `useEventChannel`, firing sonner toasts and patching React Query caches. UI components consume only the hooks. Desktop surface is an `AppSidebar` item with badge (no desktop top bar exists); mobile surface is a bell + popover in the `Layout` header.
|
||||
|
||||
**Tech Stack:** React 19, React Router v7, TanStack Query, sonner, lucide-react, shadcn/ui primitives (Popover, Switch, Dialog), Vitest + RTL.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-06-10-core-notifications-design.md` (Web UI section). Server API shipped in Phase 1.
|
||||
|
||||
Commands assume the repository root is the cwd; web commands run in `web/`. `pnpm` is the package manager.
|
||||
|
||||
---
|
||||
|
||||
## Server API contract (frozen, Phase 1)
|
||||
|
||||
The `api()` client prefixes `/api/v1` and adds auth + `X-Profile-Id` headers automatically.
|
||||
|
||||
```
|
||||
GET /notifications?unread=1&category=&cursor=&limit= → {"items": Notification[], "next_cursor": number|null}
|
||||
GET /notifications/unread-count → {"count": number}
|
||||
POST /notifications/read {"ids":[...]} | {"all":true} → 204
|
||||
POST /notifications/{id}/dismiss → 204 | 404
|
||||
GET /notifications/preferences → {"preferences":[{"category","enabled"}]}
|
||||
PUT /notifications/preferences {"preferences":[...]} → 204 | 400
|
||||
GET /admin/announcements → {"items": Announcement[]}
|
||||
POST /admin/announcements {title, body, audience:{all|user_ids|library_ids}, expires_at?} → 201
|
||||
DELETE /admin/announcements/{id} → 204 | 404
|
||||
```
|
||||
|
||||
Notification JSON: `{id, user_id, profile_id?, category, type, title, body, link?, item_id?, created_at, read_at?, expires_at?}`. Categories: `request | content | announcement | system | admin`; preference categories additionally `content_digest` (never a notification category). WS: channel `"notifications"`, event `"notification.created"`, `data` = the Notification; subscribe snapshot `data` = `{"unread_count": number}`.
|
||||
|
||||
## File map
|
||||
|
||||
| File | Action | Responsibility |
|
||||
|---|---|---|
|
||||
| `web/src/api/types.ts` | modify | `AppNotification`, `NotificationPreference`, `Announcement`, `AnnouncementAudience` types; add `"notifications"` to `EventChannel` |
|
||||
| `web/src/hooks/queries/keys.ts` | modify | `notificationKeys` factory |
|
||||
| `web/src/hooks/queries/notifications.ts` | create | all query/mutation hooks (user + admin) |
|
||||
| `web/src/hooks/useNotificationsLive.ts` | create | WS subscription → toast + cache patching |
|
||||
| `web/src/hooks/useNotificationsLive.test.ts` | create | live-hook tests |
|
||||
| `web/src/components/NotificationBell.tsx` | create | badge + popover dropdown (mobile header) |
|
||||
| `web/src/components/NotificationBell.test.tsx` | create | bell tests |
|
||||
| `web/src/components/Layout.tsx` | modify | mount bell in mobile header; mount live hook |
|
||||
| `web/src/components/AppSidebar.tsx` | modify | Notifications nav item with badge |
|
||||
| `web/src/pages/Notifications.tsx` | create | inbox page |
|
||||
| `web/src/pages/Notifications.test.tsx` | create | inbox tests |
|
||||
| `web/src/pages/settings/NotificationsSettings.tsx` | create | preference toggles |
|
||||
| `web/src/pages/SettingsLayout.tsx` | modify | nav section entry |
|
||||
| `web/src/pages/AdminAnnouncements.tsx` | create | admin CRUD page |
|
||||
| `web/src/components/AdminSidebar.tsx` | modify | announcements nav item |
|
||||
| `web/src/App.tsx` | modify | routes: `/notifications`, admin `announcements`, settings `notifications` |
|
||||
|
||||
---
|
||||
|
||||
### Task 1: API types and event channel
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/api/types.ts` (EventChannel union ~line 2198; types appended near other domain types)
|
||||
|
||||
- [ ] **Step 1: Add the types**
|
||||
|
||||
Add `"notifications"` to the `EventChannel` union. Then add (named `AppNotification` to avoid colliding with the DOM `Notification` type — verify no existing `AppNotification` symbol first with grep):
|
||||
|
||||
```typescript
|
||||
export type NotificationCategory = "request" | "content" | "announcement" | "system" | "admin";
|
||||
|
||||
export type NotificationPreferenceCategory = NotificationCategory | "content_digest";
|
||||
|
||||
export interface AppNotification {
|
||||
id: number;
|
||||
user_id: number;
|
||||
profile_id?: string;
|
||||
category: NotificationCategory;
|
||||
type: string;
|
||||
title: string;
|
||||
body: string;
|
||||
link?: string;
|
||||
item_id?: string;
|
||||
created_at: string;
|
||||
read_at?: string;
|
||||
expires_at?: string;
|
||||
}
|
||||
|
||||
export interface NotificationListResponse {
|
||||
items: AppNotification[] | null;
|
||||
next_cursor: number | null;
|
||||
}
|
||||
|
||||
export interface NotificationPreference {
|
||||
category: NotificationPreferenceCategory;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface AnnouncementAudience {
|
||||
all?: boolean;
|
||||
user_ids?: number[];
|
||||
library_ids?: number[];
|
||||
}
|
||||
|
||||
export interface Announcement {
|
||||
id: number;
|
||||
title: string;
|
||||
body: string;
|
||||
audience: AnnouncementAudience;
|
||||
created_by?: number;
|
||||
created_at: string;
|
||||
expires_at?: string;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Typecheck and commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b --noEmit 2>&1 | head -5
|
||||
git add src/api/types.ts && git commit -m "feat(web): notification API types and event channel"
|
||||
```
|
||||
|
||||
(If the repo's typecheck command differs — check package.json scripts for `typecheck`/`build` — use that; `pnpm run build` also typechecks via `tsc -b`.)
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Query keys and hooks
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/hooks/queries/keys.ts`
|
||||
- Create: `web/src/hooks/queries/notifications.ts`
|
||||
|
||||
- [ ] **Step 1: Keys** — follow the existing factory style in keys.ts:
|
||||
|
||||
```typescript
|
||||
export const notificationKeys = {
|
||||
all: ["notifications"] as const,
|
||||
list: (filters: { unread?: boolean; category?: string }) =>
|
||||
[...notificationKeys.all, "list", filters] as const,
|
||||
unreadCount: () => [...notificationKeys.all, "unread-count"] as const,
|
||||
preferences: () => [...notificationKeys.all, "preferences"] as const,
|
||||
announcements: () => ["admin", "announcements"] as const,
|
||||
};
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Hooks** — `web/src/hooks/queries/notifications.ts`. Mirror the conventions in `web/src/hooks/queries/admin/users.ts` (api client import path, toast usage, invalidation):
|
||||
|
||||
```typescript
|
||||
import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { api } from "@/api/client";
|
||||
import type {
|
||||
Announcement,
|
||||
AnnouncementAudience,
|
||||
AppNotification,
|
||||
NotificationListResponse,
|
||||
NotificationPreference,
|
||||
} from "@/api/types";
|
||||
import { notificationKeys } from "@/hooks/queries/keys";
|
||||
|
||||
const PAGE_SIZE = 25;
|
||||
|
||||
export function useNotificationsList(filters: { unread?: boolean; category?: string } = {}) {
|
||||
return useInfiniteQuery({
|
||||
queryKey: notificationKeys.list(filters),
|
||||
queryFn: async ({ pageParam }) => {
|
||||
const params = new URLSearchParams();
|
||||
params.set("limit", String(PAGE_SIZE));
|
||||
if (filters.unread) params.set("unread", "1");
|
||||
if (filters.category) params.set("category", filters.category);
|
||||
if (pageParam) params.set("cursor", String(pageParam));
|
||||
return api<NotificationListResponse>(`/notifications?${params.toString()}`);
|
||||
},
|
||||
initialPageParam: 0,
|
||||
getNextPageParam: (last) => last.next_cursor ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function useUnreadCount() {
|
||||
return useQuery({
|
||||
queryKey: notificationKeys.unreadCount(),
|
||||
queryFn: () => api<{ count: number }>("/notifications/unread-count"),
|
||||
select: (d) => d.count,
|
||||
});
|
||||
}
|
||||
|
||||
export function useMarkRead() {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (body: { ids?: number[]; all?: boolean }) =>
|
||||
api("/notifications/read", { method: "POST", body: JSON.stringify(body) }),
|
||||
onSuccess: () => {
|
||||
void queryClient.invalidateQueries({ queryKey: notificationKeys.all });
|
||||
},
|
||||
onError: (err) => {
|
||||
toast.error(err instanceof Error ? err.message : "Failed to mark read");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useDismissNotification() {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (id: number) => api(`/notifications/${id}/dismiss`, { method: "POST" }),
|
||||
onSuccess: () => {
|
||||
void queryClient.invalidateQueries({ queryKey: notificationKeys.all });
|
||||
},
|
||||
onError: (err) => {
|
||||
toast.error(err instanceof Error ? err.message : "Failed to dismiss");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useNotificationPreferences() {
|
||||
return useQuery({
|
||||
queryKey: notificationKeys.preferences(),
|
||||
queryFn: () => api<{ preferences: NotificationPreference[] }>("/notifications/preferences"),
|
||||
select: (d) => d.preferences,
|
||||
});
|
||||
}
|
||||
|
||||
export function useSetNotificationPreferences() {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (preferences: NotificationPreference[]) =>
|
||||
api("/notifications/preferences", { method: "PUT", body: JSON.stringify({ preferences }) }),
|
||||
onSuccess: () => {
|
||||
void queryClient.invalidateQueries({ queryKey: notificationKeys.preferences() });
|
||||
},
|
||||
onError: (err) => {
|
||||
toast.error(err instanceof Error ? err.message : "Failed to save preferences");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useAnnouncements() {
|
||||
return useQuery({
|
||||
queryKey: notificationKeys.announcements(),
|
||||
queryFn: () => api<{ items: Announcement[] | null }>("/admin/announcements"),
|
||||
select: (d) => d.items ?? [],
|
||||
});
|
||||
}
|
||||
|
||||
export function useCreateAnnouncement() {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (body: {
|
||||
title: string;
|
||||
body: string;
|
||||
audience: AnnouncementAudience;
|
||||
expires_at?: string;
|
||||
}) => api<Announcement>("/admin/announcements", { method: "POST", body: JSON.stringify(body) }),
|
||||
onSuccess: () => {
|
||||
toast.success("Announcement published");
|
||||
void queryClient.invalidateQueries({ queryKey: notificationKeys.announcements() });
|
||||
},
|
||||
onError: (err) => {
|
||||
toast.error(err instanceof Error ? err.message : "Failed to publish");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useDeleteAnnouncement() {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: (id: number) => api(`/admin/announcements/${id}`, { method: "DELETE" }),
|
||||
onSuccess: () => {
|
||||
toast.success("Announcement deleted");
|
||||
void queryClient.invalidateQueries({ queryKey: notificationKeys.announcements() });
|
||||
},
|
||||
onError: (err) => {
|
||||
toast.error(err instanceof Error ? err.message : "Failed to delete");
|
||||
},
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Check `api()`'s actual error behavior (does it throw on non-2xx with a message? read `web/src/api/client.ts:343-380`) and the import alias style (`@/` vs relative — match the queries directory's existing imports).
|
||||
|
||||
- [ ] **Step 3: Typecheck + commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b --noEmit 2>&1 | head -5
|
||||
git add src/hooks/queries/ && git commit -m "feat(web): notification query hooks"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Live wiring hook (WS → toasts + cache)
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/hooks/useNotificationsLive.ts`
|
||||
- Test: `web/src/hooks/useNotificationsLive.test.ts`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Mock `useEventChannel` (from `@/components/realtimeEventsContext`) to capture handlers, mock `sonner`'s toast, wrap in a QueryClientProvider. Follow the mocking style of `web/src/components/RealtimeEventsProvider.test.tsx` (vi.mock + vi.hoisted).
|
||||
|
||||
```typescript
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { renderHook } from "@testing-library/react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { capturedHandlers, toastMock } = vi.hoisted(() => ({
|
||||
capturedHandlers: {} as { onEvent?: (m: unknown) => void; onSnapshot?: (m: unknown) => void },
|
||||
toastMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/realtimeEventsContext", () => ({
|
||||
useEventChannel: (_channel: string, handlers: typeof capturedHandlers) => {
|
||||
capturedHandlers.onEvent = handlers?.onEvent;
|
||||
capturedHandlers.onSnapshot = handlers?.onSnapshot;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("sonner", () => ({ toast: toastMock }));
|
||||
|
||||
const profileState = vi.hoisted(() => ({ profile: { id: "p-1", is_child: false } }));
|
||||
vi.mock("@/hooks/useAuth", () => ({ useAuth: () => profileState }));
|
||||
|
||||
import { useNotificationsLive } from "./useNotificationsLive";
|
||||
import { notificationKeys } from "@/hooks/queries/keys";
|
||||
|
||||
function setup() {
|
||||
const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
queryClient.setQueryData(notificationKeys.unreadCount(), { count: 1 });
|
||||
renderHook(() => useNotificationsLive(), {
|
||||
wrapper: ({ children }) => (
|
||||
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
|
||||
),
|
||||
});
|
||||
return queryClient;
|
||||
}
|
||||
|
||||
function frame(data: object) {
|
||||
return { type: "event", channel: "notifications", event: "notification.created", data };
|
||||
}
|
||||
|
||||
describe("useNotificationsLive", () => {
|
||||
it("toasts and bumps unread count on notification.created", () => {
|
||||
const qc = setup();
|
||||
capturedHandlers.onEvent?.(
|
||||
frame({ id: 5, category: "request", title: "Request approved", body: "Dune", profile_id: "p-1" }),
|
||||
);
|
||||
expect(toastMock).toHaveBeenCalledWith("Request approved", expect.anything());
|
||||
expect(qc.getQueryData(notificationKeys.unreadCount())).toEqual({ count: 2 });
|
||||
});
|
||||
|
||||
it("ignores frames for another profile", () => {
|
||||
setup();
|
||||
capturedHandlers.onEvent?.(
|
||||
frame({ id: 6, category: "content", title: "New content", body: "", profile_id: "p-other" }),
|
||||
);
|
||||
expect(toastMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("suppresses restricted categories on child profiles", () => {
|
||||
profileState.profile = { id: "p-1", is_child: true };
|
||||
setup();
|
||||
capturedHandlers.onEvent?.(
|
||||
frame({ id: 7, category: "request", title: "Request approved", body: "", profile_id: undefined }),
|
||||
);
|
||||
expect(toastMock).not.toHaveBeenCalled();
|
||||
profileState.profile = { id: "p-1", is_child: false };
|
||||
});
|
||||
|
||||
it("seeds unread count from snapshot", () => {
|
||||
const qc = setup();
|
||||
capturedHandlers.onSnapshot?.({ type: "snapshot", channel: "notifications", data: { unread_count: 9 } });
|
||||
expect(qc.getQueryData(notificationKeys.unreadCount())).toEqual({ count: 9 });
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
(The test file needs `.tsx` extension if JSX is used in the wrapper — name it `useNotificationsLive.test.tsx`.)
|
||||
|
||||
- [ ] **Step 2: Run to verify failure**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/hooks/useNotificationsLive.test.tsx 2>&1 | tail -5
|
||||
```
|
||||
|
||||
Expected: FAIL — module `./useNotificationsLive` not found.
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```typescript
|
||||
import { useCallback } from "react";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import type { AppNotification } from "@/api/types";
|
||||
import { useEventChannel } from "@/components/realtimeEventsContext";
|
||||
import { notificationKeys } from "@/hooks/queries/keys";
|
||||
import { useAuth } from "@/hooks/useAuth";
|
||||
|
||||
const CHILD_HIDDEN_CATEGORIES = new Set(["request", "system", "admin"]);
|
||||
|
||||
/**
|
||||
* Subscribes to the notifications WS channel: fires a toast for live
|
||||
* notifications addressed to the active profile, keeps the unread-count
|
||||
* cache current, and seeds it from the subscribe snapshot.
|
||||
* Mount exactly once inside the authenticated layout.
|
||||
*/
|
||||
export function useNotificationsLive() {
|
||||
const queryClient = useQueryClient();
|
||||
const { profile } = useAuth();
|
||||
const profileId = profile?.id;
|
||||
const isChild = profile?.is_child ?? false;
|
||||
|
||||
const onEvent = useCallback(
|
||||
(message: unknown) => {
|
||||
const frame = message as { event?: string; data?: AppNotification };
|
||||
if (frame.event !== "notification.created" || !frame.data) return;
|
||||
const n = frame.data;
|
||||
if (n.profile_id && n.profile_id !== profileId) return;
|
||||
if (isChild && CHILD_HIDDEN_CATEGORIES.has(n.category)) return;
|
||||
|
||||
queryClient.setQueryData<{ count: number }>(notificationKeys.unreadCount(), (prev) => ({
|
||||
count: (prev?.count ?? 0) + 1,
|
||||
}));
|
||||
void queryClient.invalidateQueries({
|
||||
queryKey: [...notificationKeys.all, "list"],
|
||||
refetchType: "none",
|
||||
});
|
||||
toast(n.title, { description: n.body });
|
||||
},
|
||||
[queryClient, profileId, isChild],
|
||||
);
|
||||
|
||||
const onSnapshot = useCallback(
|
||||
(message: unknown) => {
|
||||
const frame = message as { data?: { unread_count?: number } };
|
||||
if (typeof frame.data?.unread_count !== "number") return;
|
||||
queryClient.setQueryData(notificationKeys.unreadCount(), { count: frame.data.unread_count });
|
||||
},
|
||||
[queryClient],
|
||||
);
|
||||
|
||||
useEventChannel("notifications", { onEvent, onSnapshot });
|
||||
}
|
||||
```
|
||||
|
||||
NOTE on handler identity: `useEventChannel`'s effect re-subscribes when the handlers object changes — pass a stable object. Check how existing consumers handle this (grep `useEventChannel(` usages); if they memoize the handlers object, do the same: `useMemo(() => ({ onEvent, onSnapshot }), [onEvent, onSnapshot])`.
|
||||
|
||||
The toast call shape (`toast(title, { description })`) must match the test's `expect.anything()` second arg. If `useAuth` returns a different profile field shape, adapt (read `useAuth.tsx`).
|
||||
|
||||
- [ ] **Step 4: Run tests until green**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/hooks/useNotificationsLive.test.tsx 2>&1 | tail -5
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add web/src/hooks/useNotificationsLive.* && git commit -m "feat(web): live notifications wiring (toasts, unread cache)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: NotificationBell component + Layout mount
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/components/NotificationBell.tsx`
|
||||
- Test: `web/src/components/NotificationBell.test.tsx`
|
||||
- Modify: `web/src/components/Layout.tsx` (mobile header actions ~lines 151-168; also mount `useNotificationsLive()` once in the Layout component body)
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
|
||||
```typescript
|
||||
// Mocks: @/hooks/queries/notifications (useUnreadCount → 3, useNotificationsList → two items,
|
||||
// useMarkRead → mutate spy), react-router (MemoryRouter wrapper).
|
||||
// Asserts: badge renders "3"; opening the popover (click bell button, role="button" name /notifications/i)
|
||||
// shows both item titles; "Mark all read" calls markRead with {all:true};
|
||||
// item with link renders an anchor/Link to that link.
|
||||
```
|
||||
|
||||
Write it concretely following an existing component test's structure (see `web/src/components/AppSidebar.test.tsx` for provider wrapping conventions). Badge cap: 4+ digits render "99+" when count > 99 — include one assertion with count 120.
|
||||
|
||||
- [ ] **Step 2: Implement NotificationBell**
|
||||
|
||||
```tsx
|
||||
import { Bell } from "lucide-react";
|
||||
import { Link } from "react-router";
|
||||
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
|
||||
import {
|
||||
useMarkRead,
|
||||
useNotificationsList,
|
||||
useUnreadCount,
|
||||
} from "@/hooks/queries/notifications";
|
||||
|
||||
export function NotificationBell() {
|
||||
const { data: count = 0 } = useUnreadCount();
|
||||
const list = useNotificationsList({});
|
||||
const markRead = useMarkRead();
|
||||
const items = (list.data?.pages ?? []).flatMap((p) => p.items ?? []).slice(0, 10);
|
||||
|
||||
return (
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<Button variant="ghost" size="icon" aria-label="Notifications" className="relative">
|
||||
<Bell className="h-5 w-5" />
|
||||
{count > 0 && (
|
||||
<Badge className="absolute -top-1 -right-1 h-4 min-w-4 px-1 text-[10px]">
|
||||
{count > 99 ? "99+" : count}
|
||||
</Badge>
|
||||
)}
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent align="end" className="w-80 p-0">
|
||||
<div className="flex items-center justify-between border-b px-3 py-2">
|
||||
<span className="text-sm font-medium">Notifications</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
disabled={count === 0}
|
||||
onClick={() => markRead.mutate({ all: true })}
|
||||
>
|
||||
Mark all read
|
||||
</Button>
|
||||
</div>
|
||||
<ul className="max-h-96 list-none overflow-y-auto">
|
||||
{items.length === 0 && (
|
||||
<li className="text-muted-foreground px-3 py-6 text-center text-sm">
|
||||
You're all caught up
|
||||
</li>
|
||||
)}
|
||||
{items.map((n) => (
|
||||
<li key={n.id} className={n.read_at ? "opacity-60" : ""}>
|
||||
<Link to={n.link ?? "/notifications"} className="hover:bg-accent block px-3 py-2">
|
||||
<div className="text-sm font-medium">{n.title}</div>
|
||||
{n.body && <div className="text-muted-foreground truncate text-xs">{n.body}</div>}
|
||||
</Link>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<div className="border-t px-3 py-2 text-center">
|
||||
<Link to="/notifications" className="text-primary text-sm">
|
||||
View all
|
||||
</Link>
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
Verify the ui primitives exist with these export names (`badge.tsx`, `button.tsx`, `popover.tsx` under `components/ui/`); adjust imports/props to the actual variants. Match `react-router` import name used elsewhere (the repo uses `react-router` v7 — check whether components import `Link` from `react-router` or `react-router-dom`).
|
||||
|
||||
Dropdown-open toast suppression (spec requirement): export a tiny module-level flag from NotificationBell —
|
||||
|
||||
```tsx
|
||||
let bellOpenFlag = false;
|
||||
export function isNotificationDropdownOpen() {
|
||||
return bellOpenFlag;
|
||||
}
|
||||
// in the component: <Popover onOpenChange={(open) => { bellOpenFlag = open; }}>
|
||||
```
|
||||
|
||||
and in `useNotificationsLive`'s onEvent, before toasting: `if (isNotificationDropdownOpen()) return;` (cache updates still run — only the toast is suppressed; the open dropdown shows the new item via the list invalidation). Add one live-hook test: set the flag via a popover-open simulation or export a test-only setter, assert no toast while open.
|
||||
|
||||
- [ ] **Step 3: Mount in Layout**
|
||||
|
||||
In `Layout.tsx`: call `useNotificationsLive()` at the top of the component (single mount point for the live hook), and render `<NotificationBell />` in the mobile header's right-side action cluster (next to the search icon button).
|
||||
|
||||
- [ ] **Step 4: Tests green, typecheck, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/components/NotificationBell.test.tsx 2>&1 | tail -5 && pnpm exec tsc -b --noEmit 2>&1 | head -3
|
||||
git add src/components/NotificationBell.* src/components/Layout.tsx src/hooks/
|
||||
git commit -m "feat(web): notification bell with live badge in mobile header"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Sidebar nav item with badge (desktop)
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/components/AppSidebar.tsx` (nav list — insert after the History item, ~line 645)
|
||||
- Test: extend `web/src/components/AppSidebar.test.tsx`
|
||||
|
||||
- [ ] **Step 1: Add the nav item**
|
||||
|
||||
Follow the exact ViewTransitionLink pattern of the History item (active indicator span, icon sizing, `SidebarLabel`):
|
||||
|
||||
```tsx
|
||||
<li>
|
||||
<ViewTransitionLink
|
||||
to="/notifications"
|
||||
onClick={onNavigate}
|
||||
className={navLinkClass("/notifications")}
|
||||
aria-current={isActive("/notifications") ? "page" : undefined}
|
||||
>
|
||||
{isActive("/notifications") && (
|
||||
<span
|
||||
className="absolute top-1/2 left-0 h-[18px] w-[3px] -translate-y-1/2 rounded-r-sm"
|
||||
style={{ background: "var(--primary)" }}
|
||||
/>
|
||||
)}
|
||||
<Bell className="h-[18px] w-[18px] shrink-0" />
|
||||
{unreadCount > 0 && (
|
||||
<span className="bg-primary text-primary-foreground absolute top-1 left-6 flex h-4 min-w-4 items-center justify-center rounded-full px-1 text-[10px]">
|
||||
{unreadCount > 99 ? "99+" : unreadCount}
|
||||
</span>
|
||||
)}
|
||||
<SidebarLabel show={showLabels}>Notifications</SidebarLabel>
|
||||
</ViewTransitionLink>
|
||||
</li>
|
||||
```
|
||||
|
||||
`const { data: unreadCount = 0 } = useUnreadCount();` at the top of AppSidebar (import from `@/hooks/queries/notifications`; `Bell` from lucide-react — add to the existing lucide import list at line ~56). Badge positioning: verify visually against the collapsed-sidebar state (`showLabels` false) — the absolute chip must remain visible on the icon in both states; tweak offsets if the icon container differs.
|
||||
|
||||
- [ ] **Step 2: Test**
|
||||
|
||||
Extend AppSidebar.test.tsx: mock `@/hooks/queries/notifications` `useUnreadCount` to return `{ data: 5 }`; assert the rendered sidebar contains a link to `/notifications` showing "5". Follow the file's existing mock arrangement for other hooks.
|
||||
|
||||
- [ ] **Step 3: Run, typecheck, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/components/AppSidebar.test.tsx 2>&1 | tail -4
|
||||
git add src/components/AppSidebar.* && git commit -m "feat(web): notifications sidebar item with unread badge"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Inbox page + route
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/pages/Notifications.tsx`
|
||||
- Test: `web/src/pages/Notifications.test.tsx`
|
||||
- Modify: `web/src/App.tsx` (user routes block ~line 331+)
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
|
||||
Mock the notifications query hooks. Assertions:
|
||||
- renders rows for items across categories with title/body/relative time;
|
||||
- category filter tabs (All, Requests, Content, Announcements, System — admin tab only when `useAuth().user?.role === "admin"`) change the `category` filter passed to `useNotificationsList` (capture via mock);
|
||||
- unread rows visually distinct (e.g. a `data-unread="true"` attribute — assert on it);
|
||||
- per-row dismiss button calls `useDismissNotification().mutate(id)`;
|
||||
- "Load more" appears when `hasNextPage` and calls `fetchNextPage`;
|
||||
- on mount, visible unread ids are marked read: `useMarkRead().mutate({ ids: [...] })` (assert called once with the unread ids from page one).
|
||||
|
||||
- [ ] **Step 2: Implement**
|
||||
|
||||
Page structure (use the repo's page conventions — check an existing simple page like the History/Collections page for header/container classes):
|
||||
|
||||
```tsx
|
||||
const CATEGORY_TABS: { key: string; label: string; adminOnly?: boolean }[] = [
|
||||
{ key: "", label: "All" },
|
||||
{ key: "request", label: "Requests" },
|
||||
{ key: "content", label: "Content" },
|
||||
{ key: "announcement", label: "Announcements" },
|
||||
{ key: "system", label: "System" },
|
||||
{ key: "admin", label: "Admin", adminOnly: true },
|
||||
];
|
||||
```
|
||||
|
||||
- `useState` for active category; `useNotificationsList({ category })` (omit when "");
|
||||
- mark-on-view effect:
|
||||
|
||||
```tsx
|
||||
const firstPage = list.data?.pages?.[0];
|
||||
const markRead = useMarkRead();
|
||||
const markedRef = useRef(false);
|
||||
useEffect(() => {
|
||||
if (markedRef.current || !firstPage) return;
|
||||
const unreadIds = (firstPage.items ?? []).filter((n) => !n.read_at).map((n) => n.id);
|
||||
if (unreadIds.length > 0) markRead.mutate({ ids: unreadIds });
|
||||
markedRef.current = true;
|
||||
}, [firstPage, markRead]);
|
||||
```
|
||||
|
||||
- rows: title, body, `created_at` rendered with the repo's existing relative-time helper (grep `formatDistance\|relativeTime\|timeAgo` in web/src/lib — use what exists; if nothing, use `Intl.RelativeTimeFormat` inline helper in the page);
|
||||
- dismiss button (X icon) per row; "Mark all read" header button; "Load more" via `fetchNextPage` when `hasNextPage`;
|
||||
- empty state per tab.
|
||||
|
||||
- [ ] **Step 3: Route**
|
||||
|
||||
In `App.tsx`, register alongside the other authenticated user pages (match the guard used by e.g. the favorites/history page — likely inside the layout route with profile required):
|
||||
|
||||
```tsx
|
||||
<Route path="/notifications" element={<Notifications />} />
|
||||
```
|
||||
|
||||
(Lazy-load if neighboring pages are lazy — match the file's import pattern.)
|
||||
|
||||
- [ ] **Step 4: Tests, typecheck, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/pages/Notifications.test.tsx 2>&1 | tail -5 && pnpm exec tsc -b --noEmit 2>&1 | head -3
|
||||
git add src/pages/Notifications.* src/App.tsx && git commit -m "feat(web): notifications inbox page"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Settings preferences section
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/pages/settings/NotificationsSettings.tsx`
|
||||
- Modify: `web/src/pages/SettingsLayout.tsx` (NAV_SECTIONS), `web/src/App.tsx` (settings child route)
|
||||
|
||||
- [ ] **Step 1: Implement the settings page**
|
||||
|
||||
Template: `web/src/pages/settings/PlaybackSettings.tsx` (SettingsGroup/SettingRow/Switch composition). Content:
|
||||
|
||||
```tsx
|
||||
const PREF_LABELS: Record<string, { label: string; description: string; adminOnly?: boolean }> = {
|
||||
request: { label: "Requests", description: "Updates when your requests are approved, declined, or ready" },
|
||||
content: { label: "New content", description: "New arrivals matching your watchlist, favorites, and shows in progress" },
|
||||
system: { label: "Account & security", description: "Password changes and account notices" },
|
||||
admin: { label: "Admin alerts", description: "Failed jobs and scans (admins only)", adminOnly: true },
|
||||
content_digest: { label: "Daily digest", description: "One daily summary of everything added to your libraries" },
|
||||
};
|
||||
```
|
||||
|
||||
- `useNotificationPreferences()` for current values; render a `Switch` per category in PREF_LABELS order, skipping `adminOnly` entries when `useAuth().user?.role !== "admin"`;
|
||||
- on toggle: `useSetNotificationPreferences().mutate([{ category, enabled }])` (single-category PUT — server upserts per row);
|
||||
- note under the group: "Announcements from your server admin can't be turned off."
|
||||
|
||||
- [ ] **Step 2: Register in SettingsLayout NAV_SECTIONS**
|
||||
|
||||
Add a "Notifications" section (icon `Bell`) pointing at path `notifications`, following the structure at SettingsLayout.tsx lines 39-100. Register the child route in App.tsx where other settings pages are routed (find `PlaybackSettings` route registration and mirror it).
|
||||
|
||||
- [ ] **Step 3: Test**
|
||||
|
||||
Extend `web/src/pages/SettingsLayout.test.tsx` if it asserts nav items (check); add `NotificationsSettings` rendering test only if sibling settings pages have tests (check `ls web/src/pages/settings/*.test.tsx`) — match local convention; if none exist, rely on the layout test + typecheck (note this in the commit message).
|
||||
|
||||
- [ ] **Step 4: Typecheck, test, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b --noEmit 2>&1 | head -3 && pnpm exec vitest run src/pages/SettingsLayout.test.tsx 2>&1 | tail -4
|
||||
git add src/pages/settings/NotificationsSettings.tsx src/pages/SettingsLayout.tsx src/App.tsx
|
||||
git commit -m "feat(web): notification preference toggles in settings"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Admin announcements page
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/pages/AdminAnnouncements.tsx`
|
||||
- Modify: `web/src/components/AdminSidebar.tsx` (nav items ~lines 81-134), `web/src/App.tsx` (admin routes ~lines 363-398)
|
||||
|
||||
- [ ] **Step 1: Implement the page**
|
||||
|
||||
Template: `web/src/pages/AdminUsers.tsx` for the table + dialog patterns (read it first; reuse its Table/Dialog/Button composition). Features:
|
||||
|
||||
- Table of announcements: title, audience summary ("Everyone" / "N users" / "Libraries: a, b"), created date, expires date, delete button with confirm dialog ("Deleting dismisses unread copies from user inboxes.").
|
||||
- "New announcement" button → Dialog with: title (Input, required), body (Textarea), audience radio group (Everyone / Specific users / Libraries) — for "Specific users" a multi-select of users (check how AdminUsers fetches the user list — reuse that hook; if a multi-select primitive is missing, comma-separated ids input is NOT acceptable — use checkboxes in a scrollable list), for Libraries the same against the libraries list hook (grep `useLibraries` in hooks/queries) — optional `expires_at` datetime-local input.
|
||||
- Submit via `useCreateAnnouncement()`; disable submit until title non-empty and a valid audience selection exists.
|
||||
|
||||
- [ ] **Step 2: Nav + route**
|
||||
|
||||
AdminSidebar: add item (icon `Megaphone` from lucide-react) labeled "Announcements" → `/admin/announcements`, following the existing items' structure. App.tsx admin routes: `<Route path="announcements" element={<AdminAnnouncements />} />`.
|
||||
|
||||
- [ ] **Step 3: Test**
|
||||
|
||||
`web/src/pages/AdminAnnouncements.test.tsx` — mock the three admin hooks; assert: table renders rows incl. audience summaries; create dialog validates (submit disabled with empty title); submitting with Everyone calls `useCreateAnnouncement().mutate` with `{audience: {all: true}, ...}`; delete confirm calls `useDeleteAnnouncement().mutate(id)`.
|
||||
|
||||
- [ ] **Step 4: Tests, typecheck, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/pages/AdminAnnouncements.test.tsx 2>&1 | tail -5 && pnpm exec tsc -b --noEmit 2>&1 | head -3
|
||||
git add src/pages/AdminAnnouncements.* src/components/AdminSidebar.tsx src/App.tsx
|
||||
git commit -m "feat(web): admin announcements management page"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Full verification
|
||||
|
||||
- [ ] **Step 1: Web checks**
|
||||
|
||||
```bash
|
||||
cd web && pnpm run lint && pnpm run format:check && pnpm exec vitest run 2>&1 | tail -6 && pnpm run build 2>&1 | tail -4
|
||||
```
|
||||
|
||||
All green; build also refreshes `web/dist` so the Go embed keeps working.
|
||||
|
||||
- [ ] **Step 2: Backend still green**
|
||||
|
||||
```bash
|
||||
GOWORK=off go build ./... && GOWORK=off go test ./internal/api/handlers/ 2>&1 | tail -2
|
||||
make verify-local-paths
|
||||
```
|
||||
|
||||
- [ ] **Step 3: E2E smoke against the dev stack**
|
||||
|
||||
```bash
|
||||
docker compose up -d postgres redis
|
||||
GOWORK=off go build -o /tmp/silo-web-e2e ./cmd/silo && /tmp/silo-web-e2e &
|
||||
```
|
||||
|
||||
In a browser (or report for manual check): log in, confirm the sidebar Notifications item + mobile bell render; create an announcement in /admin/announcements; verify toast fires live (WS) and badge increments without reload; open inbox, rows mark read, badge clears; toggle a preference off and verify it persists after reload.
|
||||
|
||||
- [ ] **Step 4: Commit anything outstanding; report**
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- Desktop bell+dropdown from the spec adapted to a sidebar nav item + badge (no desktop top bar exists in this app); the dropdown ships on the mobile header bell. Spec intent (always-visible unread indicator + 1-click access) preserved.
|
||||
- Toast child/profile filtering is client-side by necessity (WS frames are user-scoped; see spec Future work on claims enrichment) — `useNotificationsLive` suppresses other-profile and child-restricted frames.
|
||||
- `content_digest` rows arrive as category `content` (server stores digest as content/content.digest) — the inbox Content tab includes them; the preferences page exposes the separate `content_digest` toggle. No UI filters by `content_digest` category (final server review noted this trap).
|
||||
- Mark-on-view marks only page one's unread ids once per mount — deliberate; deeper pages mark via "Mark all read".
|
||||
- Several steps include verify-greps (ui primitive names, Link import source, relative-time helper, settings test conventions, multi-select availability) — implementers must align with what exists rather than invent.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,838 @@
|
||||
# Core Push — Web Client & Admin Config — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make push operable end-to-end from the browser: a Web Push client (service worker, subscribe, receive, per-device control, device list) plus an admin provider-config page (APNs/FCM/WebPush credentials, VAPID keygen, readiness), on top of the frozen push API.
|
||||
|
||||
**Architecture:** Five web units + two small Go endpoints. A static service worker receives pushes; a `usePushDevice` hook owns the browser subscription lifecycle; the existing notifications-settings page gains a push control + device list; a new admin page reuses the existing encrypted-settings write path for credentials. Two net-new server endpoints: VAPID keygen and a self-targeted test push.
|
||||
|
||||
**Tech Stack:** React 19, TanStack Query, sonner, the existing api client (auto device/profile headers), browser Push API / Service Worker / Notifications API, Go (chi), webpush-go (already a dep). Vitest + RTL.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-06-10-core-push-web-client-design.md`. The push registration API + delivery worker are frozen (Phase-3-server).
|
||||
|
||||
Commands assume the repository root is the cwd; web commands run in `web/` with `pnpm`. Go: `export GOWORK=off`.
|
||||
|
||||
## Frozen API the client calls
|
||||
|
||||
```
|
||||
GET /api/v1/notifications/push/webpush-key → {vapid_public_key}
|
||||
PUT /api/v1/notifications/push/device {transport, token} → 204 (needs X-Profile-Id; api client sends it)
|
||||
DELETE /api/v1/notifications/push/device → 204
|
||||
GET /api/v1/notifications/push/devices → {devices:[{device_id,name,platform,transport,push_enabled,registered_at}]}
|
||||
PUT /api/v1/notifications/push/devices/{device_id} {enabled} → 204
|
||||
GET /api/v1/admin/push/status → {apns,fcm,webpush: bool}
|
||||
PUT /api/v1/admin/settings/{key} {value} → writes an encrypted push.* key (existing)
|
||||
GET /api/v1/admin/settings/sensitive-status → {configured:[keys]} (existing)
|
||||
```
|
||||
|
||||
Net-new this plan: `POST /api/v1/admin/push/generate-vapid-keys` and `POST /api/v1/admin/push/test`.
|
||||
|
||||
## File map
|
||||
|
||||
| File | Action | Responsibility |
|
||||
|---|---|---|
|
||||
| `internal/api/handlers/push.go` | modify | add `HandleGenerateVAPIDKeys`, `HandleSendTestPush`; add a `notifier` dep |
|
||||
| `internal/api/router.go` | modify | register the two admin routes |
|
||||
| `cmd/silo/main.go` | modify | pass the notifications service into the push handler |
|
||||
| `web/public/sw.js` | create | service worker: push, notificationclick, pushsubscriptionchange |
|
||||
| `web/src/lib/pushSw.ts` | create | pure SW logic (notification options, click URL) — tested; mirrored in sw.js |
|
||||
| `web/src/lib/push.ts` | create | `urlBase64ToUint8Array`, VAPID-key cache helpers |
|
||||
| `web/src/main.tsx` | modify | register `/sw.js` when supported |
|
||||
| `web/src/api/types.ts` | modify | `PushDeviceInfo`, `VapidKeyPair` |
|
||||
| `web/src/hooks/queries/push.ts` | create | query/mutation hooks (webpush-key, devices, toggle, register/revoke, admin keygen/test) |
|
||||
| `web/src/hooks/usePushDevice.ts` | create | browser subscription lifecycle state machine |
|
||||
| `web/src/hooks/usePushDevice.test.tsx` | create | hook tests |
|
||||
| `web/src/pages/settings/NotificationsSettings.tsx` | modify | push control + device list |
|
||||
| `web/src/pages/admin-settings/PushSettings.tsx` | create | three credential cards + keygen + test |
|
||||
| `web/src/pages/admin-settings/AdminSettingsLayout.tsx` | modify | nav entry |
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Server endpoints — VAPID keygen + test push
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/api/handlers/push.go`, `internal/api/router.go`, `cmd/silo/main.go`
|
||||
- Test: `internal/api/handlers/push_test.go`
|
||||
|
||||
- [ ] **Step 1: Add a notifier dep + the two handlers**
|
||||
|
||||
In `push.go`, add a narrow interface and a field on `PushHandler`:
|
||||
|
||||
```go
|
||||
// pushNotifier creates a notification for the calling user (test push).
|
||||
type pushNotifier interface {
|
||||
CreateSystem(ctx context.Context, userID int, typ, title, body string)
|
||||
}
|
||||
```
|
||||
|
||||
Add `notifier pushNotifier` to `PushHandler` and a parameter to `NewPushHandler` (nil-tolerant — test-push 503s if absent). `*notifications.Service` satisfies `pushNotifier` (it has `CreateSystem(ctx, userID int, typ, title, body string)` from Phase-1).
|
||||
|
||||
Handlers:
|
||||
|
||||
```go
|
||||
func (h *PushHandler) HandleGenerateVAPIDKeys(w http.ResponseWriter, r *http.Request) {
|
||||
priv, pub, err := webpush.GenerateVAPIDKeys()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "failed to generate keys")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"vapid_public": pub, "vapid_private": priv})
|
||||
}
|
||||
|
||||
func (h *PushHandler) HandleSendTestPush(w http.ResponseWriter, r *http.Request) {
|
||||
if h.notifier == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "notifications unavailable")
|
||||
return
|
||||
}
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
if userID == 0 {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "auth required")
|
||||
return
|
||||
}
|
||||
h.notifier.CreateSystem(r.Context(), userID,
|
||||
"system.test_push", "Test push",
|
||||
"If you can see this on a device, push delivery is working.")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
```
|
||||
|
||||
Import `webpush "github.com/SherClockHolmes/webpush-go"`. **Verify the `GenerateVAPIDKeys()` return order** against the installed version — webpush-go returns `(privateKey, publicKey string, err error)`. Confirm by reading the vendored signature or `go doc`; if it's `(public, private, err)` swap accordingly. Get this right — swapping the keys silently breaks push.
|
||||
|
||||
- [ ] **Step 2: Routes**
|
||||
|
||||
In `router.go`, in the admin push block (next to `r.Get("/push/status", ...)`, ~line 2295, behind `deps.PushStore != nil`):
|
||||
|
||||
```go
|
||||
r.Post("/push/generate-vapid-keys", pushHandler.HandleGenerateVAPIDKeys)
|
||||
r.Post("/push/test", pushHandler.HandleSendTestPush)
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Wire the notifier**
|
||||
|
||||
Update every `NewPushHandler(...)` call site (find with grep) to pass the notifications service. In `cmd/silo/main.go` and/or `router.go` where the push handler is constructed, add `deps.NotificationsService` (or the in-scope `notificationsSvc`) as the third arg. Build will tell you the sites.
|
||||
|
||||
- [ ] **Step 4: Tests**
|
||||
|
||||
Add to `push_test.go` (reuse the package's fakes / chi-request helper from Task-13 push tests):
|
||||
|
||||
```go
|
||||
func TestPushHandleGenerateVAPIDKeys_ReturnsPair(t *testing.T) {
|
||||
h := NewPushHandler(&fakePushRegistry{}, &fakePushConfig{}, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
h.HandleGenerateVAPIDKeys(rr, httptest.NewRequest(http.MethodPost, "/", nil))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d", rr.Code)
|
||||
}
|
||||
var body map[string]string
|
||||
_ = json.Unmarshal(rr.Body.Bytes(), &body)
|
||||
if body["vapid_public"] == "" || body["vapid_private"] == "" || body["vapid_public"] == body["vapid_private"] {
|
||||
t.Fatalf("bad key pair: %+v", body)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeNotifier struct{ called bool; userID int; typ string }
|
||||
|
||||
func (f *fakeNotifier) CreateSystem(_ context.Context, userID int, typ, _, _ string) {
|
||||
f.called = true
|
||||
f.userID = userID
|
||||
f.typ = typ
|
||||
}
|
||||
|
||||
func TestPushHandleSendTestPush_CreatesForCaller(t *testing.T) {
|
||||
n := &fakeNotifier{}
|
||||
h := NewPushHandler(&fakePushRegistry{}, &fakePushConfig{}, n)
|
||||
r := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
r = r.WithContext(apimw.SetUserID(r.Context(), 7)) // use the package's claims-injection helper if SetUserID differs
|
||||
rr := httptest.NewRecorder()
|
||||
h.HandleSendTestPush(rr, r)
|
||||
if rr.Code != http.StatusAccepted || !n.called || n.userID != 7 || n.typ != "system.test_push" {
|
||||
t.Fatalf("unexpected: code=%d notifier=%+v", rr.Code, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushHandleSendTestPush_NilNotifier503(t *testing.T) {
|
||||
h := NewPushHandler(&fakePushRegistry{}, &fakePushConfig{}, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
h.HandleSendTestPush(rr, httptest.NewRequest(http.MethodPost, "/", nil))
|
||||
if rr.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("code = %d", rr.Code)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Check how the existing push tests inject the user id into context (Task-13 used a claims/`apimw` helper — mirror it; `apimw.SetUserID` may not exist, use whatever the package's tests already use to set `GetUserID`).
|
||||
|
||||
- [ ] **Step 5: Build, test, commit**
|
||||
|
||||
```bash
|
||||
GOWORK=off go build ./... && GOWORK=off go test ./internal/api/handlers/ -run TestPush -v 2>&1 | tail -8 && GOWORK=off go vet ./internal/api/handlers/
|
||||
git add internal/api/handlers/push.go internal/api/handlers/push_test.go internal/api/router.go cmd/silo/main.go
|
||||
git commit -m "feat(push): admin VAPID keygen and self-test endpoints"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Service worker + registration + helpers
|
||||
|
||||
**Files:**
|
||||
- Create: `web/public/sw.js`, `web/src/lib/pushSw.ts`, `web/src/lib/push.ts`
|
||||
- Test: `web/src/lib/pushSw.test.ts`, `web/src/lib/push.test.ts`
|
||||
- Modify: `web/src/main.tsx`
|
||||
|
||||
- [ ] **Step 1: Pure SW logic (tested) — `web/src/lib/pushSw.ts`**
|
||||
|
||||
```ts
|
||||
export interface PushPayload {
|
||||
id?: number;
|
||||
title?: string;
|
||||
body?: string;
|
||||
link?: string;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
export interface SwNotification {
|
||||
title: string;
|
||||
options: { body: string; data: { link: string }; tag: string };
|
||||
}
|
||||
|
||||
/** Maps a raw push payload to showNotification args; tolerant of partial/missing fields. */
|
||||
export function buildNotification(payload: PushPayload | null): SwNotification {
|
||||
const p = payload ?? {};
|
||||
const link = p.link && p.link.length > 0 ? p.link : "/notifications";
|
||||
const tag = p.id != null ? `n${p.id}` : "n";
|
||||
return {
|
||||
title: p.title && p.title.length > 0 ? p.title : "Silo",
|
||||
options: { body: p.body ?? "", data: { link }, tag },
|
||||
};
|
||||
}
|
||||
|
||||
/** The URL a notification click should open/focus. */
|
||||
export function clickUrl(data: unknown): string {
|
||||
const d = (data ?? {}) as { link?: string };
|
||||
return d.link && d.link.length > 0 ? d.link : "/notifications";
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Test it — `web/src/lib/pushSw.test.ts`**
|
||||
|
||||
```ts
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildNotification, clickUrl } from "./pushSw";
|
||||
|
||||
describe("buildNotification", () => {
|
||||
it("maps a full payload", () => {
|
||||
const n = buildNotification({ id: 5, title: "Approved", body: "Dune", link: "/requests" });
|
||||
expect(n.title).toBe("Approved");
|
||||
expect(n.options).toEqual({ body: "Dune", data: { link: "/requests" }, tag: "n5" });
|
||||
});
|
||||
it("falls back on missing fields", () => {
|
||||
const n = buildNotification(null);
|
||||
expect(n.title).toBe("Silo");
|
||||
expect(n.options.body).toBe("");
|
||||
expect(n.options.data.link).toBe("/notifications");
|
||||
expect(n.options.tag).toBe("n");
|
||||
});
|
||||
});
|
||||
|
||||
describe("clickUrl", () => {
|
||||
it("uses link when present", () => expect(clickUrl({ link: "/items/9" })).toBe("/items/9"));
|
||||
it("falls back to inbox", () => expect(clickUrl({})).toBe("/notifications"));
|
||||
});
|
||||
```
|
||||
|
||||
Run: `pnpm exec vitest run src/lib/pushSw.test.ts` → green.
|
||||
|
||||
- [ ] **Step 3: The static service worker — `web/public/sw.js`**
|
||||
|
||||
Self-contained (served statically, not bundled). Mirror the pure logic exactly (the canonical, tested source is `pushSw.ts`):
|
||||
|
||||
```js
|
||||
// web/public/sw.js — static service worker for Web Push.
|
||||
// NOTE: the buildNotification/clickUrl logic mirrors src/lib/pushSw.ts (the
|
||||
// tested source of truth). Keep them in sync.
|
||||
|
||||
function buildNotification(payload) {
|
||||
const p = payload || {};
|
||||
const link = p.link && p.link.length > 0 ? p.link : "/notifications";
|
||||
const tag = p.id != null ? "n" + p.id : "n";
|
||||
return {
|
||||
title: p.title && p.title.length > 0 ? p.title : "Silo",
|
||||
options: { body: p.body || "", data: { link: link }, tag: tag },
|
||||
};
|
||||
}
|
||||
function clickUrl(data) {
|
||||
const d = data || {};
|
||||
return d.link && d.link.length > 0 ? d.link : "/notifications";
|
||||
}
|
||||
|
||||
self.addEventListener("push", (event) => {
|
||||
let payload = null;
|
||||
try {
|
||||
payload = event.data ? event.data.json() : null;
|
||||
} catch (e) {
|
||||
payload = null;
|
||||
}
|
||||
const n = buildNotification(payload);
|
||||
event.waitUntil(self.registration.showNotification(n.title, n.options));
|
||||
});
|
||||
|
||||
self.addEventListener("notificationclick", (event) => {
|
||||
event.notification.close();
|
||||
const url = clickUrl(event.notification.data);
|
||||
event.waitUntil(
|
||||
self.clients.matchAll({ type: "window", includeUncontrolled: true }).then((wins) => {
|
||||
for (const c of wins) {
|
||||
if ("focus" in c) {
|
||||
c.postMessage({ type: "notification-click", link: url });
|
||||
return c.focus();
|
||||
}
|
||||
}
|
||||
return self.clients.openWindow(url);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
self.addEventListener("pushsubscriptionchange", (event) => {
|
||||
// Best-effort re-subscribe using the cached VAPID key; the app re-registers
|
||||
// the new subscription on next load if this fails.
|
||||
event.waitUntil(
|
||||
(async () => {
|
||||
try {
|
||||
const cache = await caches.open("silo-push");
|
||||
const res = await cache.match("vapid-public-key");
|
||||
if (!res) return;
|
||||
const vapid = await res.text();
|
||||
const sub = await self.registration.pushManager.subscribe({
|
||||
userVisibleOnly: true,
|
||||
applicationServerKey: vapid,
|
||||
});
|
||||
await fetch("/api/v1/notifications/push/device", {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
credentials: "include",
|
||||
body: JSON.stringify({ transport: "webpush", token: JSON.stringify(sub.toJSON()) }),
|
||||
});
|
||||
} catch (e) {
|
||||
// swallow; app re-registers on next load
|
||||
}
|
||||
})(),
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
Note: the `pushsubscriptionchange` re-register relies on cookie auth (`credentials: include`) and the device/profile headers it can't set from the SW — it is genuinely best-effort and the app heals it on next load (documented in the spec). `applicationServerKey` accepts a base64url string in modern browsers; if the target browsers need a Uint8Array here, the app-side path (Task 4) converts it — the SW path is the fallback and may no-op on stricter browsers, which is acceptable.
|
||||
|
||||
- [ ] **Step 4: App-side helpers — `web/src/lib/push.ts`**
|
||||
|
||||
```ts
|
||||
/** Convert a base64url VAPID public key to the Uint8Array applicationServerKey wants. */
|
||||
export function urlBase64ToUint8Array(base64: string): Uint8Array {
|
||||
const padding = "=".repeat((4 - (base64.length % 4)) % 4);
|
||||
const b64 = (base64 + padding).replace(/-/g, "+").replace(/_/g, "/");
|
||||
const raw = atob(b64);
|
||||
const out = new Uint8Array(raw.length);
|
||||
for (let i = 0; i < raw.length; i++) out[i] = raw.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Cache the raw VAPID public key so the SW can re-subscribe without the app. */
|
||||
export async function cacheVapidKey(base64: string): Promise<void> {
|
||||
if (!("caches" in self)) return;
|
||||
const cache = await caches.open("silo-push");
|
||||
await cache.put("vapid-public-key", new Response(base64));
|
||||
}
|
||||
|
||||
export function pushSupported(): boolean {
|
||||
return (
|
||||
typeof navigator !== "undefined" &&
|
||||
"serviceWorker" in navigator &&
|
||||
typeof window !== "undefined" &&
|
||||
"PushManager" in window &&
|
||||
"Notification" in window
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Test the converter — `web/src/lib/push.test.ts`**
|
||||
|
||||
```ts
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { urlBase64ToUint8Array } from "./push";
|
||||
|
||||
describe("urlBase64ToUint8Array", () => {
|
||||
it("decodes a url-safe base64 key to bytes", () => {
|
||||
// "AQID" → [1,2,3]
|
||||
expect(Array.from(urlBase64ToUint8Array("AQID"))).toEqual([1, 2, 3]);
|
||||
});
|
||||
it("handles url-safe chars and missing padding", () => {
|
||||
const out = urlBase64ToUint8Array("a-_9"); // '-'→'+', '_'→'/'
|
||||
expect(out.length).toBe(3);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Register the SW — `web/src/main.tsx`**
|
||||
|
||||
After `createRoot(...).render(...)`, add:
|
||||
|
||||
```tsx
|
||||
if ("serviceWorker" in navigator && "PushManager" in window) {
|
||||
window.addEventListener("load", () => {
|
||||
navigator.serviceWorker.register("/sw.js").catch(() => {
|
||||
// registration failure is non-fatal; push just won't be available
|
||||
});
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run, build, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/lib/pushSw.test.ts src/lib/push.test.ts 2>&1 | tail -5 && pnpm exec tsc -b --noEmit 2>&1 | head -3
|
||||
git add web/public/sw.js web/src/lib/pushSw.ts web/src/lib/pushSw.test.ts web/src/lib/push.ts web/src/lib/push.test.ts web/src/main.tsx
|
||||
git commit -m "feat(web): web push service worker, registration, and helpers"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Push query hooks + types
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/api/types.ts`
|
||||
- Create: `web/src/hooks/queries/push.ts`
|
||||
- Modify: `web/src/hooks/queries/keys.ts` (add `pushKeys`)
|
||||
|
||||
- [ ] **Step 1: Types** — append to `types.ts`:
|
||||
|
||||
```ts
|
||||
export interface PushDeviceInfo {
|
||||
device_id: string;
|
||||
name: string;
|
||||
platform: string;
|
||||
transport: string;
|
||||
push_enabled: boolean;
|
||||
registered_at?: string;
|
||||
}
|
||||
|
||||
export interface VapidKeyPair {
|
||||
vapid_public: string;
|
||||
vapid_private: string;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Keys** — in `keys.ts`:
|
||||
|
||||
```ts
|
||||
export const pushKeys = {
|
||||
all: ["push"] as const,
|
||||
webpushKey: () => [...pushKeys.all, "webpush-key"] as const,
|
||||
devices: () => [...pushKeys.all, "devices"] as const,
|
||||
status: () => [...pushKeys.all, "status"] as const,
|
||||
};
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Hooks — `web/src/hooks/queries/push.ts`**
|
||||
|
||||
Mirror `admin/users.ts` conventions (api import, toast, invalidation).
|
||||
|
||||
```ts
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { api } from "@/api/client";
|
||||
import type { PushDeviceInfo, VapidKeyPair } from "@/api/types";
|
||||
import { pushKeys } from "@/hooks/queries/keys";
|
||||
|
||||
export function useWebPushPublicKey() {
|
||||
return useQuery({
|
||||
queryKey: pushKeys.webpushKey(),
|
||||
queryFn: () => api<{ vapid_public_key: string }>("/notifications/push/webpush-key"),
|
||||
select: (d) => d.vapid_public_key,
|
||||
staleTime: 5 * 60 * 1000,
|
||||
});
|
||||
}
|
||||
|
||||
export function usePushDevices() {
|
||||
return useQuery({
|
||||
queryKey: pushKeys.devices(),
|
||||
queryFn: () => api<{ devices: PushDeviceInfo[] | null }>("/notifications/push/devices"),
|
||||
select: (d) => d.devices ?? [],
|
||||
});
|
||||
}
|
||||
|
||||
export function useTogglePushDevice() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: ({ deviceId, enabled }: { deviceId: string; enabled: boolean }) =>
|
||||
api(`/notifications/push/devices/${deviceId}`, {
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ enabled }),
|
||||
}),
|
||||
onSuccess: () => void qc.invalidateQueries({ queryKey: pushKeys.devices() }),
|
||||
onError: (err) => toast.error(err instanceof Error ? err.message : "Failed to update device"),
|
||||
});
|
||||
}
|
||||
|
||||
export function usePushStatus() {
|
||||
return useQuery({
|
||||
queryKey: pushKeys.status(),
|
||||
queryFn: () => api<{ apns: boolean; fcm: boolean; webpush: boolean }>("/admin/push/status"),
|
||||
});
|
||||
}
|
||||
|
||||
export function useGenerateVapidKeys() {
|
||||
return useMutation({
|
||||
mutationFn: () => api<VapidKeyPair>("/admin/push/generate-vapid-keys", { method: "POST" }),
|
||||
onError: (err) => toast.error(err instanceof Error ? err.message : "Failed to generate keys"),
|
||||
});
|
||||
}
|
||||
|
||||
export function useSendTestPush() {
|
||||
return useMutation({
|
||||
mutationFn: () => api("/admin/push/test", { method: "POST" }),
|
||||
onSuccess: () => toast.success("Test push queued — watch for a banner"),
|
||||
onError: (err) => toast.error(err instanceof Error ? err.message : "Failed to send test"),
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
(Register/revoke device calls live inside `usePushDevice` (Task 4) since they're tied to the subscription, not standalone query hooks.)
|
||||
|
||||
- [ ] **Step 4: Typecheck + commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b --noEmit 2>&1 | head -3
|
||||
git add web/src/api/types.ts web/src/hooks/queries/keys.ts web/src/hooks/queries/push.ts
|
||||
git commit -m "feat(web): push query hooks and types"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: `usePushDevice` lifecycle hook
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/hooks/usePushDevice.ts`, `web/src/hooks/usePushDevice.test.tsx`
|
||||
|
||||
- [ ] **Step 1: Failing tests**
|
||||
|
||||
Mock `navigator.serviceWorker`, `window.PushManager`, `Notification`, the api client, and the push helpers. Assert the status machine and the enable/disable flows.
|
||||
|
||||
```tsx
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { act, renderHook, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { apiMock, cacheVapidMock } = vi.hoisted(() => ({
|
||||
apiMock: vi.fn(),
|
||||
cacheVapidMock: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/api/client", () => ({ api: apiMock }));
|
||||
vi.mock("@/lib/push", async (orig) => ({
|
||||
...(await orig<typeof import("@/lib/push")>()),
|
||||
cacheVapidKey: cacheVapidMock,
|
||||
}));
|
||||
|
||||
import { usePushDevice } from "./usePushDevice";
|
||||
|
||||
function wrap() {
|
||||
const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
return ({ children }: { children: React.ReactNode }) => (
|
||||
<QueryClientProvider client={qc}>{children}</QueryClientProvider>
|
||||
);
|
||||
}
|
||||
|
||||
let subscribeMock: ReturnType<typeof vi.fn>;
|
||||
let getSubMock: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
apiMock.mockReset();
|
||||
cacheVapidMock.mockReset();
|
||||
subscribeMock = vi.fn(async () => ({ toJSON: () => ({ endpoint: "https://x", keys: {} }) }));
|
||||
getSubMock = vi.fn(async () => null);
|
||||
(globalThis as any).PushManager = function () {};
|
||||
(navigator as any).serviceWorker = {
|
||||
register: vi.fn(),
|
||||
ready: Promise.resolve({ pushManager: { subscribe: subscribeMock, getSubscription: getSubMock } }),
|
||||
};
|
||||
(globalThis as any).Notification = { permission: "default", requestPermission: vi.fn(async () => "granted") };
|
||||
});
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
describe("usePushDevice", () => {
|
||||
it("reports unsupported when PushManager missing", () => {
|
||||
delete (globalThis as any).PushManager;
|
||||
const { result } = renderHook(() => usePushDevice(), { wrapper: wrap() });
|
||||
expect(result.current.status).toBe("unsupported");
|
||||
});
|
||||
|
||||
it("reports blocked when permission denied", () => {
|
||||
(globalThis as any).Notification.permission = "denied";
|
||||
const { result } = renderHook(() => usePushDevice(), { wrapper: wrap() });
|
||||
expect(result.current.status).toBe("blocked");
|
||||
});
|
||||
|
||||
it("enable(): permission → key → subscribe → PUT device, caches key, status on", async () => {
|
||||
apiMock.mockImplementation((path: string) => {
|
||||
if (path.includes("webpush-key")) return Promise.resolve({ vapid_public_key: "AQID" });
|
||||
return Promise.resolve(undefined); // PUT device
|
||||
});
|
||||
const { result } = renderHook(() => usePushDevice(), { wrapper: wrap() });
|
||||
await act(async () => { await result.current.enable(); });
|
||||
expect(subscribeMock).toHaveBeenCalledOnce();
|
||||
expect(cacheVapidMock).toHaveBeenCalledWith("AQID");
|
||||
const putCall = apiMock.mock.calls.find((c) => c[0] === "/notifications/push/device");
|
||||
expect(putCall?.[1]).toMatchObject({ method: "PUT" });
|
||||
expect(JSON.parse(putCall![1].body)).toMatchObject({ transport: "webpush" });
|
||||
await waitFor(() => expect(result.current.status).toBe("on"));
|
||||
});
|
||||
|
||||
it("enable(): permission denied short-circuits, no subscribe", async () => {
|
||||
(globalThis as any).Notification.requestPermission = vi.fn(async () => "denied");
|
||||
const { result } = renderHook(() => usePushDevice(), { wrapper: wrap() });
|
||||
await act(async () => { await result.current.enable(); });
|
||||
expect(subscribeMock).not.toHaveBeenCalled();
|
||||
expect(result.current.status).toBe("blocked");
|
||||
});
|
||||
|
||||
it("disable(): unsubscribe + DELETE, status off", async () => {
|
||||
const unsub = vi.fn(async () => true);
|
||||
getSubMock.mockResolvedValue({ unsubscribe: unsub });
|
||||
apiMock.mockResolvedValue(undefined);
|
||||
const { result } = renderHook(() => usePushDevice(), { wrapper: wrap() });
|
||||
await act(async () => { await result.current.disable(); });
|
||||
expect(unsub).toHaveBeenCalled();
|
||||
const del = apiMock.mock.calls.find((c) => c[0] === "/notifications/push/device" && c[1]?.method === "DELETE");
|
||||
expect(del).toBeTruthy();
|
||||
await waitFor(() => expect(result.current.status).toBe("off"));
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run, verify fail**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/hooks/usePushDevice.test.tsx 2>&1 | tail -6
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Implement — `web/src/hooks/usePushDevice.ts`**
|
||||
|
||||
```ts
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import { api } from "@/api/client";
|
||||
import { cacheVapidKey, pushSupported, urlBase64ToUint8Array } from "@/lib/push";
|
||||
|
||||
export type PushDeviceStatus = "unsupported" | "blocked" | "off" | "on" | "pending";
|
||||
|
||||
export function usePushDevice() {
|
||||
const [status, setStatus] = useState<PushDeviceStatus>("pending");
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
if (!pushSupported()) {
|
||||
setStatus("unsupported");
|
||||
return;
|
||||
}
|
||||
if (Notification.permission === "denied") {
|
||||
setStatus("blocked");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const reg = await navigator.serviceWorker.ready;
|
||||
const sub = await reg.pushManager.getSubscription();
|
||||
setStatus(sub ? "on" : "off");
|
||||
} catch {
|
||||
setStatus("off");
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
const enable = useCallback(async () => {
|
||||
if (!pushSupported()) {
|
||||
setStatus("unsupported");
|
||||
return;
|
||||
}
|
||||
setStatus("pending");
|
||||
const perm = await Notification.requestPermission();
|
||||
if (perm !== "granted") {
|
||||
setStatus(perm === "denied" ? "blocked" : "off");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const reg = await navigator.serviceWorker.ready;
|
||||
const { vapid_public_key } = await api<{ vapid_public_key: string }>(
|
||||
"/notifications/push/webpush-key",
|
||||
);
|
||||
if (!vapid_public_key) {
|
||||
setStatus("off");
|
||||
return;
|
||||
}
|
||||
await cacheVapidKey(vapid_public_key);
|
||||
const sub = await reg.pushManager.subscribe({
|
||||
userVisibleOnly: true,
|
||||
applicationServerKey: urlBase64ToUint8Array(vapid_public_key),
|
||||
});
|
||||
await api("/notifications/push/device", {
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ transport: "webpush", token: JSON.stringify(sub.toJSON()) }),
|
||||
});
|
||||
setStatus("on");
|
||||
} catch {
|
||||
setStatus("off");
|
||||
}
|
||||
}, []);
|
||||
|
||||
const disable = useCallback(async () => {
|
||||
setStatus("pending");
|
||||
try {
|
||||
const reg = await navigator.serviceWorker.ready;
|
||||
const sub = await reg.pushManager.getSubscription();
|
||||
if (sub) await sub.unsubscribe();
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
try {
|
||||
await api("/notifications/push/device", { method: "DELETE" });
|
||||
} catch {
|
||||
/* server may already lack the row */
|
||||
}
|
||||
setStatus("off");
|
||||
}, []);
|
||||
|
||||
return { status, enable, disable, refresh };
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run green, typecheck, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec vitest run src/hooks/usePushDevice.test.tsx 2>&1 | tail -6 && pnpm exec tsc -b --noEmit 2>&1 | head -3
|
||||
git add web/src/hooks/usePushDevice.ts web/src/hooks/usePushDevice.test.tsx
|
||||
git commit -m "feat(web): usePushDevice subscription lifecycle hook"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: NotificationsSettings — push control + device list
|
||||
|
||||
**Files:**
|
||||
- Modify: `web/src/pages/settings/NotificationsSettings.tsx`
|
||||
- Test: extend the page's test if one exists (check `ls web/src/pages/settings/*.test.tsx`)
|
||||
|
||||
- [ ] **Step 1: Add a push section**
|
||||
|
||||
Read the current `NotificationsSettings.tsx` (it uses `SettingsGroup`/`SettingRow`/`Switch`). Add, below the category-preference group, a new "Push notifications" group:
|
||||
|
||||
- A row driven by `usePushDevice()`:
|
||||
- `status === "unsupported"` → static text "This browser doesn't support push notifications."
|
||||
- `status === "blocked"` → text "Notifications are blocked for this site. Re-enable them in your browser's site settings."
|
||||
- else → a `Switch` checked when `status === "on"`, disabled when `status === "pending"`, `onCheckedChange` calls `enable()` / `disable()`.
|
||||
- Below it, a device list from `usePushDevices()`: one row per device (`name || device_id`, `platform`/`transport`, `registered_at`), each with a `Switch` bound to `useTogglePushDevice().mutate({ deviceId, enabled })` reflecting `push_enabled`. Empty state: "No devices registered for push yet."
|
||||
|
||||
Match the page's existing component composition and classes. Keep `content_digest` and the existing toggles untouched.
|
||||
|
||||
- [ ] **Step 2: Test (if the page is tested)**
|
||||
|
||||
If `NotificationsSettings.test.tsx` exists, mock `usePushDevice` (return `{status:"off", enable, disable}`) and `usePushDevices` (two devices); assert: toggling the device-level switch calls `useTogglePushDevice().mutate` with the device id; the main push switch calls `enable` when off. If no sibling test exists, rely on typecheck + the hook tests and note it in the commit.
|
||||
|
||||
- [ ] **Step 3: Typecheck, test, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b --noEmit 2>&1 | head -3 && pnpm exec vitest run src/pages/settings 2>&1 | tail -4
|
||||
git add web/src/pages/settings/NotificationsSettings.tsx web/src/pages/settings/*.test.tsx
|
||||
git commit -m "feat(web): push on-this-device control and device list in settings"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Admin PushSettings page + nav
|
||||
|
||||
**Files:**
|
||||
- Create: `web/src/pages/admin-settings/PushSettings.tsx`
|
||||
- Modify: `web/src/pages/admin-settings/AdminSettingsLayout.tsx`
|
||||
- Test: `web/src/pages/admin-settings/PushSettings.test.tsx`
|
||||
|
||||
- [ ] **Step 1: Build the page**
|
||||
|
||||
Template: `IntegrationsSettings.tsx` (credential card = `CredentialStatus` badge + `SettingField` + `useUpdateServerSetting` + `useAdminSensitiveStatus`). Three cards. Header badge per card from `usePushStatus()`.
|
||||
|
||||
- **Web Push card** (leads): fields `push.webpush.vapid_public`, `push.webpush.vapid_private`, `push.webpush.subject`. A "Generate keys" button → `useGenerateVapidKeys().mutateAsync()` fills the public + private field state (local `useState`, not auto-saved). Subject field hint: `mailto:you@example.com`. Each field saves via `updateSetting.mutateAsync({ key, value })`. A "Send test push" button → `useSendTestPush().mutate()`.
|
||||
- **APNs card**: `push.apns.p8_key` (textarea via `SettingField` multiline if supported, else a `<textarea>`), `push.apns.key_id`, `push.apns.team_id`, `push.apns.bundle_id`.
|
||||
- **FCM card**: `push.fcm.service_account_json` (textarea).
|
||||
|
||||
`configured` per field from `new Set(sensitive?.configured ?? []).has(key)`. Page header + intro like IntegrationsSettings.
|
||||
|
||||
- [ ] **Step 2: Register nav**
|
||||
|
||||
In `AdminSettingsLayout.tsx`, add to the "Connections" group's `items` (import the component + a lucide icon, e.g. `BellRing`):
|
||||
|
||||
```tsx
|
||||
{ id: "push", label: "Push Notifications", icon: BellRing, component: PushSettings },
|
||||
```
|
||||
|
||||
(`id: "push"` is stable URL state.)
|
||||
|
||||
- [ ] **Step 3: Test — `PushSettings.test.tsx`**
|
||||
|
||||
Mock `useUpdateServerSetting` (capture mutate args), `useAdminSensitiveStatus` (some keys configured), `usePushStatus` (webpush true), `useGenerateVapidKeys` (returns a pair), `useSendTestPush`. Assert: three cards render; webpush badge shows ready; saving the VAPID public field calls `updateSetting.mutateAsync({ key: "push.webpush.vapid_public", value })`; "Generate keys" populates the public/private fields; "Send test push" calls the mutation. Follow the admin-settings test conventions (check a sibling `*.test.tsx` in that dir; if none, write a minimal RTL test with the mocks).
|
||||
|
||||
- [ ] **Step 4: Typecheck, test, build, commit**
|
||||
|
||||
```bash
|
||||
cd web && pnpm exec tsc -b --noEmit 2>&1 | head -3 && pnpm exec vitest run src/pages/admin-settings 2>&1 | tail -5
|
||||
git add web/src/pages/admin-settings/PushSettings.tsx web/src/pages/admin-settings/PushSettings.test.tsx web/src/pages/admin-settings/AdminSettingsLayout.tsx
|
||||
git commit -m "feat(web): admin push provider config page"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Full verification + manual smoke
|
||||
|
||||
- [ ] **Step 1: Web gates**
|
||||
|
||||
```bash
|
||||
cd web && pnpm run lint 2>&1 | tail -3 && pnpm run format:check 2>&1 | tail -2 && pnpm exec vitest run 2>&1 | tail -5 && pnpm run build 2>&1 | tail -3
|
||||
```
|
||||
|
||||
All green (lint warnings allowed, errors not). Fix any new-code lint/format issues (prettier --write the new files if format:check flags them). `build` also confirms `public/sw.js` is copied to `dist/` (Vite copies `public/` verbatim).
|
||||
|
||||
- [ ] **Step 2: Backend gates**
|
||||
|
||||
```bash
|
||||
GOWORK=off go build ./... && GOWORK=off go test ./internal/api/handlers/ -run TestPush 2>&1 | tail -3 && make verify-local-paths
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Manual browser smoke (documented; requires a real browser over HTTPS or localhost)**
|
||||
|
||||
Service workers + Push require a secure context — `localhost` qualifies. Build the server with the web assets and run on a localhost port.
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
```bash
|
||||
cd web && pnpm run build && cd ..
|
||||
GOWORK=off go build -o ./bin/silo-pushweb ./cmd/silo && (PORT=18080 JF_PORT=18096 ./bin/silo-pushweb > ./silo-pushweb.log 2>&1 &)
|
||||
```
|
||||
|
||||
Then in a Chromium browser at `http://localhost:18080`:
|
||||
1. Log in as admin. Go to Admin → Settings → Push Notifications. Click "Generate keys", Save the VAPID public/private + a `mailto:` subject. Confirm the Web Push badge flips to "Ready" (and `GET /admin/push/status` → webpush:true).
|
||||
2. Go to Settings → Notifications. Toggle "Push on this device" on → grant the browser permission prompt. Confirm a row appears in the device list and the toggle stays on.
|
||||
3. Click "Send test push" on the admin page (or create an announcement). Within ~30–45s (grace + worker tick) a desktop notification banner appears. Close the app tab first to prove background delivery.
|
||||
4. Click the banner → the app opens/focuses at the deep link.
|
||||
5. Toggle push off → confirm `DELETE` and the device drops from the list.
|
||||
|
||||
Record the outcome. (If a real browser isn't available in the harness, run steps 1–2 via curl to confirm the API wiring — register a webpush device, confirm `push_deliveries` row + status — and note the visual steps as operator-verified.)
|
||||
|
||||
- [ ] **Step 4: Commit any verification fixes; report.** Work stays on `feat/core-notifications-server` (or a dedicated branch off it).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes (resolved during writing)
|
||||
|
||||
- `webpush.GenerateVAPIDKeys()` returns `(privateKey, publicKey, err)` in webpush-go — Task 1 Step 1 flags verifying the order so the keys aren't swapped.
|
||||
- The SW's pure logic is duplicated between `public/sw.js` (untestable static file) and the tested `src/lib/pushSw.ts`; the duplication is small and intentional, with sw.js annotated as mirroring the tested source. The reviewer should diff the two functions for drift.
|
||||
- `usePushDevice` owns register/revoke (tied to the live subscription); standalone query hooks (`push.ts`) cover read/list/toggle/admin. No overlap.
|
||||
- Test push reuses `notifications.Service.CreateSystem` (Phase-1) → a real system notification → exercises the full enqueue→worker→transport pipeline to the caller's own devices. No bespoke push path.
|
||||
- Admin credential writes use the existing `PUT /admin/settings/{key}` (encrypted, redacted) — no new write endpoint; only keygen + test are net-new server surface.
|
||||
- iOS Safari web push (installed-PWA only) is handled by `pushSupported()` returning false in a normal tab → `unsupported` state; documented, not specially coded.
|
||||
- Several steps embed verify-greps (NewPushHandler call sites, GenerateVAPIDKeys order, SettingField multiline support, sibling test conventions, the user-id context injection helper in push_test.go) — align with reality.
|
||||
@@ -1,13 +1,14 @@
|
||||
# Audiobooks Absorption — Discovery Findings
|
||||
|
||||
Produced by sub-plan 1, Task 1. Locks data-model and integration
|
||||
decisions for migrations 139–142 and downstream sub-plans.
|
||||
decisions for the audiobook foundation migrations and downstream sub-plans.
|
||||
|
||||
## D1 — Next migration number
|
||||
|
||||
Highest existing migration: 138_search_number_word_normalization.up.sql
|
||||
Next available number: 139
|
||||
Sub-plan 1 will use migrations 139 through 142.
|
||||
Original snapshot: `138_search_number_word_normalization.up.sql` was the
|
||||
highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
|
||||
## D2 — `media_libraries` kind/type column
|
||||
|
||||
@@ -16,10 +17,10 @@ The spec refers to `media_libraries` but the actual table is `media_folders`.
|
||||
library content. Current values in production: `movies`, `series`, `mixed`.
|
||||
|
||||
Existing column 'type' (text) discriminates library content on `media_folders`.
|
||||
Task 4 (migration 141) should ADD the value `audiobooks` to the type vocabulary
|
||||
Task 4 (migration 159) should ADD the value `audiobooks` to the type vocabulary
|
||||
rather than add a new column. The audiobook scanner branch will set
|
||||
`media_folders.type = 'audiobooks'` for audiobook libraries. No schema change
|
||||
needed for the column itself; migration 141 becomes a no-op DDL migration that
|
||||
needed for the column itself; migration 159 becomes a no-op DDL migration that
|
||||
documents the new allowed value and adds any supporting indexes if needed.
|
||||
|
||||
No CHECK constraint or enum enforces the `type` column values, so adding
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
# Audiobook Player QoL Improvements
|
||||
|
||||
Commands assume the repository root is the cwd. All paths are repository-relative.
|
||||
|
||||
## Goal
|
||||
|
||||
Bring the web audiobook player to feature parity with best-in-class audiobook apps
|
||||
(Audiobookshelf, Audible, BookPlayer, Prologue) on the quality-of-life axis. The current
|
||||
surface architecture (persistent mini bar + expandable Now Listening view, playback
|
||||
survives navigation) is already the de-facto standard and stays as-is.
|
||||
|
||||
Explicitly **out of scope**: Media Session / lock-screen integration, bookmarks (needs a
|
||||
server endpoint — follow-up), offline/download, queue.
|
||||
|
||||
## Features
|
||||
|
||||
1. **Keyboard shortcuts** — space/K play-pause, arrow skips, volume, chapter nav, speed step.
|
||||
2. **Configurable skip intervals** — asymmetric defaults (back 10s, forward 30s), user-tunable.
|
||||
3. **Expanded speed control** — 0.5×–3.0× with fine stepping, presets, and per-book memory.
|
||||
4. **Chapter prev/next buttons** — alongside the seconds-skip buttons.
|
||||
5. **Volume control** — reuse the shared `VolumeControl`, persisted like the video player.
|
||||
6. **Smart rewind** — auto-rewind on resume, scaled by how long playback was paused.
|
||||
7. **Time remaining at current speed** — real-clock remaining display in Now Listening.
|
||||
|
||||
No server/API changes are required; everything is web-frontend only, so no Android/Apple
|
||||
client coordination is needed. (Per-book speed memory is device-local in v1; syncing it
|
||||
server-side via profile settings is a noted follow-up.)
|
||||
|
||||
## Existing seams (read these first)
|
||||
|
||||
| File | Relevance |
|
||||
| --- | --- |
|
||||
| `web/src/pages/audiobooks/player/useAudiobookPlayback.ts` | Playback engine. `togglePlay` (~line 499), `skip` (~534), `setRate` (~541), returned API (~690+). All engine work lands here. |
|
||||
| `web/src/pages/audiobooks/player/MiniBar.tsx` | Bottom bar. Hardcoded `SKIP_*_SECONDS = 30` and `PLAYBACK_RATES` at lines 10–12. |
|
||||
| `web/src/pages/audiobooks/player/NowListening.tsx` | Full-screen view. Same hardcoded constants; dead "More" button (lines 50–56) becomes the settings menu trigger; remaining-time toggle at lines 33–37. |
|
||||
| `web/src/pages/audiobooks/player/AudiobookPlayer.tsx` | Wrapper that owns mini/now-listening mode — mount point for the keyboard shortcuts hook. |
|
||||
| `web/src/player/components/VolumeControl.tsx` | Existing slider + `getPersistedVolume`/`persistVolume` helpers (shared `player-volume`/`player-muted` storage keys). Styled for dark video overlay — needs a surface-toned variant. |
|
||||
| `web/src/player/components/SpeedMenu.tsx` | Current preset-only menu (shared with video player). Audiobook gets a richer `SpeedControl`; video keeps `SpeedMenu` untouched. |
|
||||
| `web/src/player/hooks/useKeyboardShortcuts.ts` | Video player's shortcut hook — pattern to mirror (input-field guard, single document listener). |
|
||||
| `web/src/utils/storage.ts` | Typed localStorage wrapper — add new keys here, never call `localStorage` directly. |
|
||||
| `web/src/player/components/SeekBar.tsx`, `CircleButton.tsx`, `SleepTimerMenu.tsx`, `ChaptersMenu.tsx` | Established player design language; reuse, don't fork. |
|
||||
|
||||
## Design language
|
||||
|
||||
These are additions to an existing, refined system — extend it, don't restyle it:
|
||||
|
||||
- **Surfaces**: mini bar is a themed surface (`bg-background`, `border-t`, token colors).
|
||||
Popovers anchored to it use the established dark-glass treatment
|
||||
(`bg-black/90 backdrop-blur-sm rounded-lg shadow-xl`, white/75 text, `data-active` rows)
|
||||
exactly as `SpeedMenu`/`SleepTimerMenu` do today.
|
||||
- **Type**: time and rate values are always `font-mono`/`tabular-nums` so nothing shifts as
|
||||
digits tick. The new speed readout (`1.45×`) must reserve width for two decimals.
|
||||
- **Buttons**: all transport controls are `CircleButton` (`sm` secondary flanks, `md`/`lg`
|
||||
primary center). Chapter prev/next use `SkipBack`/`SkipForward` lucide glyphs at the same
|
||||
stroke weight (1.6) as the rotate icons, so the cluster reads as one family.
|
||||
- **Micro-interactions**, small and purposeful:
|
||||
- Skip buttons: a quick ~18° icon flick in the skip direction on press
|
||||
(CSS transform transition, ~150ms ease-out) — confirms the action without a toast.
|
||||
- Speed stepper: value crossfades (opacity 120ms) on change; the active preset chip gets
|
||||
the same `bg-white/5 text-white` treatment as current menu rows.
|
||||
- Smart rewind: when it fires, the seek bar playhead animates the small jump back rather
|
||||
than teleporting (SeekBar already animates position; verify it holds for ≤30s deltas).
|
||||
- **Accessibility**: every new control keeps the existing patterns — `aria-label`,
|
||||
`aria-expanded`/`aria-haspopup` on triggers, roving focus + Escape in menus (copy from
|
||||
`SleepTimerMenu`), `role="slider"` semantics from `VolumeControl`/`SeekBar`.
|
||||
|
||||
## Phase 1 — Engine + preferences model
|
||||
|
||||
### 1a. Preferences hook
|
||||
|
||||
New `web/src/pages/audiobooks/player/useAudiobookPrefs.ts`:
|
||||
|
||||
- Add storage keys: `AUDIOBOOK_SKIP_BACK`, `AUDIOBOOK_SKIP_FORWARD`,
|
||||
`AUDIOBOOK_SMART_REWIND` (bool), `AUDIOBOOK_RATES` (JSON map) in `web/src/utils/storage.ts`.
|
||||
- Exposes `{ skipBack, skipForward, smartRewind, setSkipBack, setSkipForward, setSmartRewind }`.
|
||||
- Defaults: back **10s**, forward **30s** (the asymmetric convention — small "what did she
|
||||
say?" hops back, bigger hops forward), smart rewind **on**.
|
||||
- Allowed skip values: 5, 10, 15, 30, 45, 60, 90 (two digits max keeps `SkipIcon`'s inset
|
||||
number legible).
|
||||
- Per-book rate map: single JSON object `{ [contentId]: rate }` under `AUDIOBOOK_RATES`,
|
||||
LRU-capped at 50 entries (store `{ rate, at }` and evict oldest on insert).
|
||||
Helpers `getBookRate(contentId)` / `setBookRate(contentId, rate)` live here too.
|
||||
|
||||
### 1b. Engine extensions (`useAudiobookPlayback.ts`)
|
||||
|
||||
- **Volume/mute**: initialize from `getPersistedVolume()`; apply to the `<audio>` element in
|
||||
an effect; expose `volume`, `muted`, `setVolume`, `setMuted`; persist via `persistVolume`.
|
||||
Sharing the video player's keys is intentional — one volume preference per device.
|
||||
- **Per-book rate**: on mount, initialize `rate` from `getBookRate(contentId)` (fallback 1);
|
||||
in `setRate`, also `setBookRate`. The hook needs `contentId` passed in (it already
|
||||
receives the files/options object — extend that input).
|
||||
- **Chapter navigation**: expose `nextChapter()` and `prevChapter()`.
|
||||
- `next`: seek to start of the chapter after `currentChapter`; no-op at the last chapter.
|
||||
- `prev`: if more than 3s into the current chapter, seek to its start; otherwise seek to
|
||||
the previous chapter's start (the universal music/audiobook convention).
|
||||
- **Smart rewind**: record `pausedAtRef = performance.now()` whenever playback pauses (in
|
||||
the `pause` event handler so it also catches OS-initiated pauses). On resume, before
|
||||
`audio.play()`, compute rewind from pause duration and `seekTo(currentTime - rewind)`:
|
||||
- < 10s paused → 0s; < 1min → 3s; < 10min → 10s; < 1h → 20s; ≥ 1h → 30s.
|
||||
- Clamp at 0; skip entirely when the smart-rewind pref is off or the resume immediately
|
||||
follows a user seek (seeking sets a short suppress flag so explicit jumps aren't undone).
|
||||
- Also apply on **cold resume**: when starting with `initialPositionSeconds > 0` from the
|
||||
detail page, rewind by the same schedule using the age of the saved progress if the
|
||||
API exposes an updated-at timestamp; otherwise apply a flat 10s. Keep this logic in one
|
||||
pure function `smartRewindSeconds(pauseMs)` so it's unit-testable.
|
||||
- **Skip intervals**: no engine change — `skip(delta)` already takes a signed delta; the UI
|
||||
passes the configured values.
|
||||
|
||||
### 1c. Tests
|
||||
|
||||
Vitest, colocated like existing `*.test.tsx`:
|
||||
|
||||
- `smartRewindSeconds` schedule boundaries.
|
||||
- Per-book rate LRU (cap, eviction, malformed-JSON tolerance).
|
||||
- `prevChapter` 3s threshold behavior and first/last chapter edges
|
||||
(chapter math is pure — test against `buildChapterList` fixtures from
|
||||
`web/src/lib/audiobooks/chapters.ts`).
|
||||
|
||||
## Phase 2 — Controls UI
|
||||
|
||||
### 2a. SpeedControl (new, audiobook-specific)
|
||||
|
||||
New `web/src/pages/audiobooks/player/SpeedControl.tsx`, replacing `SpeedMenu` usage in both
|
||||
audiobook views (video player keeps `SpeedMenu`):
|
||||
|
||||
- Trigger: same `player-utility-btn` showing the current rate (`1.45×`, tabular-nums).
|
||||
- Popover (dark-glass, bottom-anchored like today):
|
||||
- A stepper row: `−` / big mono readout / `+`, stepping **0.05** per tap, clamped to
|
||||
**0.5–3.0**. Press-and-hold repeats (250ms initial, 80ms repeat).
|
||||
- A preset chip row: 1× · 1.25× · 1.5× · 1.75× · 2× · 2.5× · 3×.
|
||||
- Footnote line: "Remembered for this book" (quiet `text-white/40`, 11px) — makes the
|
||||
per-book memory discoverable instead of magical.
|
||||
- Keyboard: ArrowUp/Down step inside the popover; Escape closes (copy menu plumbing from
|
||||
`SleepTimerMenu`).
|
||||
|
||||
### 2b. Chapter prev/next buttons
|
||||
|
||||
- Add `SkipBack`/`SkipForward` `CircleButton size="sm" variant="secondary"` on the outside
|
||||
of the transport cluster in both `MiniBar` and `NowListening`:
|
||||
`⏮ ↺10 ▶ ↻30 ⏭`. Wire to `prevChapter`/`nextChapter`; render only when
|
||||
`playback.chapters.length > 0`; disable at the ends (prev stays enabled mid-chapter).
|
||||
- In `MiniBar`, hide the chapter buttons below `sm:` — the 3-column grid is already tight
|
||||
on phones and the chapters menu remains available.
|
||||
|
||||
### 2c. Volume
|
||||
|
||||
- Add a `tone` prop to `web/src/player/components/VolumeControl.tsx`
|
||||
(`"overlay"` = current white-on-dark, `"surface"` = theme tokens: track `bg-muted`,
|
||||
fill `bg-foreground`, focus ring `ring-ring`). Default `"overlay"` so the video player is
|
||||
untouched.
|
||||
- Mount in `MiniBar`'s right cluster (before the sleep timer) at `md:` and up, and in
|
||||
`NowListening`'s utility row. Wire to the new engine `volume`/`muted` state.
|
||||
|
||||
### 2d. Player settings menu
|
||||
|
||||
New `web/src/pages/audiobooks/player/PlayerSettingsMenu.tsx` (dark-glass popover):
|
||||
|
||||
- **Skip back** / **Skip forward**: rows of small value chips (5/10/15/30/45/60/90s).
|
||||
- **Smart rewind**: toggle row with a one-line description ("Backs up a little after a pause").
|
||||
- Triggers: the currently-dead "More" (`MoreHorizontal`) button in `NowListening`
|
||||
(lines 50–56) and a matching trigger in `MiniBar`'s right cluster.
|
||||
- `MiniBar`/`NowListening` read `useAudiobookPrefs()` and pass the configured seconds to
|
||||
the skip buttons and `SkipIcon` (which already renders a dynamic number) — delete the
|
||||
`SKIP_*_SECONDS` constants.
|
||||
|
||||
## Phase 3 — Shortcuts + time display + polish
|
||||
|
||||
### 3a. Keyboard shortcuts
|
||||
|
||||
New `web/src/pages/audiobooks/player/useAudiobookKeyboardShortcuts.ts`, mounted in
|
||||
`AudiobookPlayer.tsx` (so it's active in both mini and expanded modes, on every route):
|
||||
|
||||
| Key | Action |
|
||||
| --- | --- |
|
||||
| Space / K | play-pause |
|
||||
| ← / → | skip by configured back/forward seconds |
|
||||
| ↑ / ↓ | volume ±5% |
|
||||
| M | mute toggle |
|
||||
| N / P | next / previous chapter |
|
||||
| Shift+. / Shift+, | speed +0.05 / −0.05 (YouTube convention) |
|
||||
| E | expand / collapse Now Listening |
|
||||
| Esc | collapse Now Listening (expanded mode only) |
|
||||
|
||||
- Mirror `useKeyboardShortcuts.ts` guards: ignore when target is input/textarea/
|
||||
contentEditable, single document listener, cleanup on unmount.
|
||||
- Additional guard: **do nothing while a video session is active** — the video player binds
|
||||
the same keys. Check `WatchPlaybackProvider` state (or simply whether a `<video>` watch
|
||||
route is mounted) and bail; the audiobook bar is backgrounded in that case anyway. Space
|
||||
must also not fire when a button has focus (let the focused control handle Enter/Space).
|
||||
- Add `title` tooltips with the shortcut hint to transport buttons
|
||||
(e.g. `Back 10 seconds (←)`) for discoverability.
|
||||
|
||||
### 3b. Time remaining at current speed
|
||||
|
||||
In `NowListening`, the right time label currently toggles total ↔ remaining. Make it cycle
|
||||
three states (persist last choice in component state only):
|
||||
|
||||
1. total (`12:04:00`)
|
||||
2. remaining (`−3:21:09`)
|
||||
3. remaining at speed (`−2:14:06 at 1.5×`) — `remaining / rate`, hidden (skipped in the
|
||||
cycle) when `rate === 1`.
|
||||
|
||||
Keep the `data-testid="now-listening-right-time"` hook and update its test.
|
||||
|
||||
### 3c. Polish pass
|
||||
|
||||
- Skip-flick and speed-crossfade micro-interactions from the design section.
|
||||
- Verify mini bar layout on 360px-wide viewports with all new controls (chapter buttons and
|
||||
volume hidden, settings menu accessible).
|
||||
- `cd web && pnpm run lint && pnpm run format:check`; run the player test suites.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- `useAudiobookPlayback.ts` is 707 lines and growing — extract smart rewind + prefs-coupled
|
||||
logic into small pure modules (`smartRewind.ts`) rather than inflating the hook; consider
|
||||
splitting volume/rate concerns into a `useAudioElementSettings` helper if the hook passes
|
||||
~800 lines.
|
||||
- Smart rewind must not fight the 10s progress reporter: rewinding on resume changes
|
||||
`currentTime`, which triggers a progress write — that's correct (the rewound position is
|
||||
the truth), just confirm no oscillation with the suppress-after-seek flag.
|
||||
- Per-book rate uses `contentId`; switching narrators navigates to a different `contentId`,
|
||||
so each narration remembers its own speed — acceptable, arguably desirable.
|
||||
- Cross-device sync of prefs/per-book speed is a candidate follow-up once a profile
|
||||
settings endpoint exists; the `useAudiobookPrefs` API is the seam to swap storage behind.
|
||||
@@ -41,9 +41,10 @@ Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
### Out
|
||||
|
||||
- Audiobook requests flow (`silo-plugin-audiobook-requests` untouched).
|
||||
- Smart collections, share links, content restrictions, custom metadata providers.
|
||||
- Smart collections, share links, content restrictions, metadata-provider integration.
|
||||
- Embedding-powered "similar books" recommender.
|
||||
- External enrichment (OpenLibrary / Google Books) — deferred; file tags only for v1.
|
||||
- External enrichment through the audiobook metadata plugin/provider — deferred;
|
||||
scanner extracts local tags and identifiers for v1.
|
||||
- Group listening / cowatch parity for audiobooks (silo already has cowatch for
|
||||
video; revisit later if needed).
|
||||
|
||||
@@ -53,7 +54,7 @@ Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
|
||||
New top-level Go package: `internal/audiobooks/`.
|
||||
|
||||
```
|
||||
```text
|
||||
internal/audiobooks/
|
||||
abs/ ← ported from plugin's internal/abs/ (~6.8k LOC)
|
||||
ABS-compatible REST handlers, mounted under /abs/* and the
|
||||
@@ -90,7 +91,9 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
- ~7.4k LOC ported from the plugin.
|
||||
- ~1.5–2k LOC new TypeScript/TSX in silo's SPA.
|
||||
- ~400 LOC of new Go glue (scanner branches, router wiring, service.go).
|
||||
- Two new SQL migrations (134, 135) and at most one column-add migration.
|
||||
- Three new SQL migrations in the current stack (`147_abs_sessions`,
|
||||
`157_podcast_feeds`, `160_audiobooks_feature_flag`) plus one documented
|
||||
no-op migration for the already-present media-folder type column.
|
||||
|
||||
## Data model
|
||||
|
||||
@@ -111,9 +114,9 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### New tables (two migrations)
|
||||
|
||||
1. **`abs_sessions`** (migration 134) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
1. **`abs_sessions`** (migration 147) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token hash, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token_hash`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
|
||||
2. **`podcast_feeds`** (migration 135) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
2. **`podcast_feeds`** (migration 157) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
|
||||
### Schema-touch column add
|
||||
|
||||
@@ -124,8 +127,8 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### Plugin migrations NOT ported
|
||||
|
||||
Smart collections, share links, embeddings, content restrictions, custom
|
||||
metadata providers, request provider, file cache, listening stats aggregates,
|
||||
Smart collections, share links, embeddings, content restrictions,
|
||||
metadata-provider integration, request provider, file cache, listening stats aggregates,
|
||||
reading goals, notification prefs, standalone-mode tables, recommender
|
||||
embeddings. All represent features dropped from scope.
|
||||
|
||||
@@ -225,8 +228,9 @@ and directors. No new code, just two more role string constants.
|
||||
|
||||
### External enrichment
|
||||
|
||||
Deferred. The plugin's `enrich/` package (OpenLibrary / Google Books lookup)
|
||||
is not ported in v1. File tags only. Add later if needed.
|
||||
Deferred. The scanner extracts local tags and identity hints only. Canonical
|
||||
enrichment remains the responsibility of the audiobook metadata plugin/provider
|
||||
path and is not ported in this foundation pass.
|
||||
|
||||
### Scheduled task
|
||||
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# Resilient Library Deletion Design
|
||||
|
||||
## Goal
|
||||
|
||||
Make `delete_library` survive concurrent database writers and large libraries. Today it deletes an entire library inside a single multi-minute transaction; on a large library this deadlocks against ordinary background writes and rolls back, making zero progress. Rewrite the deletion to run in small, autocommitted batches with per-batch deadlock retry, so it completes reliably, holds only short locks, and resumes cleanly if interrupted.
|
||||
|
||||
## Background
|
||||
|
||||
A `delete_library` admin job repeatedly failed with:
|
||||
|
||||
```
|
||||
deleting orphaned items: ERROR: deadlock detected (SQLSTATE 40P01)
|
||||
```
|
||||
|
||||
Diagnosis on a library of ~249K items / ~339K media files:
|
||||
|
||||
- The whole deletion runs in one transaction in `FolderRepository.DeleteWithStats` (`internal/catalog/folder_repo.go`): collect orphan IDs, drop the folder (CASCADE), then `DELETE FROM media_items WHERE content_id = ANY($1)`.
|
||||
- `media_items` cascades to 15 child tables, so a bulk delete holds a very large lock set for minutes.
|
||||
- The Postgres deadlock log shows the cycle: the delete cascade into `media_item_provider_ids` versus a concurrent `INSERT INTO media_item_provider_ids` from a routine background task. Each ~7-minute attempt was guaranteed to overlap an ordinary write and lose.
|
||||
- Because it is one transaction, every failure rolls back completely: no progress, not resumable.
|
||||
|
||||
The fix targets the deletion mechanism only. Coordinating or pausing background writers is explicitly out of scope (see Out of Scope) — a batched, retrying delete is robust against *any* concurrent writer, not just today's tasks.
|
||||
|
||||
## Approach
|
||||
|
||||
Replace the single transaction with a sequence of phases. Each phase deletes in bounded batches, and each batch is one autocommit statement wrapped in a deadlock-retry helper. Short lock windows make deadlocks rare; when one occurs, only that small batch retries.
|
||||
|
||||
Scope of the rewrite:
|
||||
|
||||
- Rewrite `FolderRepository.DeleteWithStats` in `internal/catalog/folder_repo.go`.
|
||||
- Add an unexported `retryOnDeadlock` helper in the `catalog` package.
|
||||
- `FolderRepository.Delete` (thin wrapper), the `DeleteFolderStats` struct, and the `delete_library` executor in `internal/adminjob/library_delete.go` keep their current signatures and behavior. S3 image-cleanup queueing and generated-home-section cleanup are untouched.
|
||||
|
||||
## Deletion phases
|
||||
|
||||
`DeleteWithStats(ctx, id, progress)` performs, in order:
|
||||
|
||||
### Phase 0 — preflight (reads only)
|
||||
|
||||
- Load the folder name; return `ErrFolderNotFound` if the row is gone.
|
||||
- `COUNT` media files (`WHERE media_folder_id = $1`), item links (`WHERE media_folder_id = $1`), and orphans, to populate the result payload and the progress denominator.
|
||||
|
||||
### Phase 1 — orphaned `media_items` (batched loop)
|
||||
|
||||
Repeat until no rows are returned:
|
||||
|
||||
```sql
|
||||
SELECT mil.content_id
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.media_folder_id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries other
|
||||
WHERE other.content_id = mil.content_id
|
||||
AND other.media_folder_id <> $1
|
||||
)
|
||||
LIMIT $2 -- orphanDeleteBatch
|
||||
```
|
||||
|
||||
For each returned batch: collect S3 image dirs for those IDs (existing `collectImageDirs`, accumulated into a deduped set), then
|
||||
|
||||
```sql
|
||||
DELETE FROM media_items WHERE content_id = ANY($1)
|
||||
```
|
||||
|
||||
The FK cascade removes each item's `media_item_libraries` (including this folder's), `media_item_provider_ids`, episodes, seasons, embeddings, and the rest of the 15 child tables. Deleting orphan items first also clears their junction rows, so the next iteration's detection query naturally advances. Orphan detection must run before Phase 3 touches shared junctions, so the "only in this folder" test stays valid.
|
||||
|
||||
### Phase 2 — folder `media_files` (batched loop)
|
||||
|
||||
`media_files` is folder-tied (`media_folder_id` FK, `ON DELETE CASCADE`), not item-tied, so it is deleted by folder. Repeat until a batch deletes fewer than the batch size:
|
||||
|
||||
```sql
|
||||
DELETE FROM media_files
|
||||
WHERE id IN (
|
||||
SELECT id FROM media_files WHERE media_folder_id = $1 LIMIT $2 -- folderChildDeleteBatch
|
||||
)
|
||||
```
|
||||
|
||||
### Phase 3 — remaining shared-item junctions (batched loop)
|
||||
|
||||
After Phase 1, the only `media_item_libraries` rows left for this folder belong to items that also live in other folders. Delete them in batches (dropping the membership only, never the shared item):
|
||||
|
||||
```sql
|
||||
DELETE FROM media_item_libraries
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM media_item_libraries WHERE media_folder_id = $1 LIMIT $2 -- folderChildDeleteBatch
|
||||
)
|
||||
```
|
||||
|
||||
### Phase 4 — folder row
|
||||
|
||||
```sql
|
||||
DELETE FROM media_folders WHERE id = $1
|
||||
```
|
||||
|
||||
Now lightweight. Tolerates 0 rows affected (a resumed re-run may already have removed it) — it does not re-raise `ErrFolderNotFound` here. The folder row is deleted **last**, so a partial deletion leaves the library present-but-disabled and a re-run continues.
|
||||
|
||||
## Deadlock-retry helper
|
||||
|
||||
New unexported helper in the `catalog` package:
|
||||
|
||||
```go
|
||||
func retryOnDeadlock(ctx context.Context, op func() error) error
|
||||
```
|
||||
|
||||
- Retries when the error is a `*pgconn.PgError` with code `40P01` (deadlock) or `40001` (serialization failure).
|
||||
- Up to 5 attempts, exponential backoff starting at 50ms, doubling each retry.
|
||||
- Honors `ctx` cancellation between attempts (returns `ctx.Err()`).
|
||||
- Any non-retryable error returns immediately; the final retryable error is returned after the cap.
|
||||
|
||||
Every batch statement runs through `retryOnDeadlock`. It has a single caller today, so it stays package-local; it can be promoted to a shared DB utility if a second caller appears.
|
||||
|
||||
## Batch sizes
|
||||
|
||||
Named constants in `internal/catalog/folder_repo.go`, tunable:
|
||||
|
||||
- `orphanDeleteBatch = 1000` — small, because each `media_items` row cascades across 15 tables.
|
||||
- `folderChildDeleteBatch = 5000` — for the lighter folder-scoped `media_files` and junction deletes.
|
||||
|
||||
## Transaction model, stats, progress
|
||||
|
||||
- No surrounding transaction. Each batch is a single `pool.Exec` (its own implicit transaction), mirroring the existing batched-cleanup pattern in `internal/opslog/cleanup.go`. This is the core change that eliminates the multi-minute lock window.
|
||||
- Stats: `MediaFiles` and `MediaItemLinks` come from the Phase 0 counts; `OrphanedItems` accumulates across Phase 1 batches; `OrphanedImageDirs` is the accumulated deduped dir set. Image-dir volume for the affected workloads is modest; the deduped set is the bounded-memory point of note.
|
||||
- Progress: the callback reports orphan items processed against the Phase 0 orphan total, plus phase-transition messages. The admin-job runner's existing 10s heartbeat keeps the job alive across the drain.
|
||||
|
||||
## Error handling & resumability
|
||||
|
||||
- Because batches autocommit, a failure or cancellation persists partial progress. There is no rollback of hundreds of thousands of rows and no long lock hold.
|
||||
- The job is resumable by re-running it: the folder still exists (deleted last), and Phase 1 re-detects whatever orphans remain.
|
||||
- New items inserted concurrently during deletion are picked up by the loop's re-detection.
|
||||
|
||||
## Testing
|
||||
|
||||
The `catalog` package uses pure-Go unit tests; there is no Postgres integration harness, so tests avoid a live database.
|
||||
|
||||
- **`retryOnDeadlock` unit test:** a fake `op` returns a `*pgconn.PgError{Code: "40P01"}` N times then `nil` — assert it retries, respects the attempt cap, backs off, and returns immediately on a non-retryable error; a separate case asserts `ctx` cancellation stops retries.
|
||||
- **Batch-loop unit test:** extract the "loop until a batch is under the batch size" iteration behind a small injected exec function so the looping/chunking logic is testable without Postgres.
|
||||
- **Integration verification on dev:** deploy and re-trigger the deletion against the real stuck library; confirm it drains to completion via application logs and database row counts. This is the integration test, given the absence of a catalog DB harness. (Commands assume the repository root is the cwd and follow the deployment-debugging runbook.)
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Pausing or coordinating background writers (e.g., having metadata/scan tasks skip disabled or pending-deletion libraries). The batched, retrying delete is robust against any concurrent writer; writer coordination is a separate potential improvement.
|
||||
- Changing the admin-job runner, the executor's image-cleanup queueing, or home-section cleanup.
|
||||
- Schema or FK changes.
|
||||
@@ -0,0 +1,141 @@
|
||||
# Calendar Presets & Personalized Default — Design
|
||||
|
||||
- **Date:** 2026-05-29
|
||||
- **Status:** Approved (design); ready for implementation planning
|
||||
- **Scope:** `silo-server` (Go backend + web admin UI). Client follow-ups in `silo-android` / `silo-apple` are additive and optional.
|
||||
- **Note:** All paths are repository-relative. Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
The calendar works but is not practical. It defaults to *every* upcoming airing in the catalog, so it is noisy and undifferentiated — the user has to mentally filter out shows they don't care about. We want a Simkl-style preset switcher with a personalized default, powered by the server's real watch data rather than a manually curated list.
|
||||
|
||||
## Goals
|
||||
|
||||
- Default the calendar to a personalized **Following** view (shows the profile actually engages with).
|
||||
- Offer a small set of presets, including a **server-wide** popularity view and an **external trending** view.
|
||||
- Keep it cheap under load (hundreds of concurrent users): presets must **filter, not compute**.
|
||||
- Make "what do I still need to watch this week" visible via a watched overlay.
|
||||
- Preserve backward compatibility for existing clients.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No cross-user caching of the windowed airing query yet (phased — add only if metrics show strain).
|
||||
- No pulling in external content that isn't in the library (e.g. a followed show whose next season hasn't been added). The calendar stays catalog-scoped.
|
||||
- No change to the week-view layout, timezone handling, or event ordering (recently tuned; left intact).
|
||||
- No region/country or "For Kids" presets (considered and cut).
|
||||
|
||||
## Presets
|
||||
|
||||
The selector replaces today's `All / Favorites / Watchlist` toggles.
|
||||
|
||||
| Preset | Shows | Source |
|
||||
|---|---|---|
|
||||
| **Following** *(default)* | Upcoming airings for series the profile has watched, favorited, or watchlisted | per-profile: `user_favorites` ∪ `user_watchlist` ∪ watched-series rollup |
|
||||
| **Popular here** | Airings for what is most-watched server-wide | cached `recommendation_cache` (`RecTypePopular`, global key) |
|
||||
| **Trending** | Airings for externally-trending shows that exist in the library | `trending_discover_snapshots` |
|
||||
| **Everything** | All catalog airings (today's behavior) | no restriction |
|
||||
|
||||
Per-profile vs server-wide is split cleanly across two presets: **Following** is personal (this profile), **Popular here** is the server-wide aggregate.
|
||||
|
||||
## Core principle: filter, don't compute
|
||||
|
||||
Every preset resolves to **the windowed airing candidate list ∩ a small id-set**. No preset triggers a fresh aggregation at request time.
|
||||
|
||||
1. **Step 1 — windowed candidate fetch** (unchanged from today): the 3-branch UNION in `internal/catalog/calendar_repo.go` over movies/episodes/season-premieres, scoped only by library access + content rating, across the ±2-day-padded window. Its result depends only on the access signature, not on viewer identity, so it remains cacheable across users later.
|
||||
2. **Step 2 — intersect with one id-set**, chosen by preset:
|
||||
- **Following** → resolve the profile's followed id-set once per request, then filter `itemIDExpr = ANY($set)`.
|
||||
- **Popular / Trending** → read the already-cached global id-set and intersect.
|
||||
- **Everything** → no restriction.
|
||||
3. **Step 3 — group by day, sort by viewer-local wall-clock time, respond** (unchanged).
|
||||
|
||||
The only heavy SQL is Step 1, which already runs today for every calendar load. Presets add a bounded id-set intersection, never a new aggregation.
|
||||
|
||||
### Unify the filter mechanism
|
||||
|
||||
All personal/preset filters resolve to "an id-set, then `itemIDExpr = ANY($set)`". This replaces the special-cased favorites/watchlist `EXISTS` subqueries in `appendPersonalFilterClause` with a single code path (favorites and watchlist become id-sets resolved from their tables — they are subsets of Following). One mechanism is easier to reason about and removes duplicated clause-building.
|
||||
|
||||
`itemIDExpr` continues to be `mi.content_id` for movies and `e.series_id` / `s.series_id` for episodes / season premieres, so a watched/favorited **movie** matches by its own content_id while a followed **series** matches by series id.
|
||||
|
||||
### Following id-set
|
||||
|
||||
Resolved per request as the DISTINCT union of:
|
||||
|
||||
- `SELECT media_item_id FROM user_favorites WHERE user_id = $u AND profile_id = $p`
|
||||
- `SELECT media_item_id FROM user_watchlist WHERE user_id = $u AND profile_id = $p`
|
||||
- `SELECT DISTINCT COALESCE(e.series_id, wp.media_item_id) FROM user_watch_progress wp LEFT JOIN episodes e ON e.content_id = wp.media_item_id WHERE wp.user_id = $u AND wp.profile_id = $p`
|
||||
|
||||
"Engaged" = the presence of a `user_watch_progress` row (you've started it); no completion threshold required for Following. The rollup mirrors the established pattern in `GetPopularItems` (`internal/recommendations/repo.go`) — reuse that rollup expression rather than duplicate it.
|
||||
|
||||
The candidate list (Step 1) is small (one week of airings), so intersecting against a bounded array is cheap. Caveat: a pathologically large followed-set (a profile that has watched thousands of distinct series) makes the array large; if that ever shows up as hot, switch the Following preset to correlated `EXISTS` against the user's tables. Not building that now.
|
||||
|
||||
### Popular / Trending id-sets
|
||||
|
||||
- **Popular**: `GetRecommendationCache(ctx, GlobalCacheUserID, GlobalCacheProfileID, RecTypePopular, "")` → `[]ScoredItem`; use `MediaItemID` values (already series-level via the `GetPopularItems` rollup). Refreshed by the recommendations worker on a TTL; readers never aggregate live.
|
||||
- **Trending**: `TrendingSnapshotRepository.Get(ctx, source, window)` → `TrendingSnapshot.ContentIDs` (pre-resolved to catalog content_ids, viewer-agnostic). The calendar reads the canonical default snapshot (`tmdb` / `week`). Unioning multiple sources/windows is possible but out of scope.
|
||||
|
||||
**Access is preserved automatically:** Popular/Trending id-sets are viewer-agnostic and may include items a given profile cannot access. Because we intersect them with the access-scoped Step-1 candidates (library scoping + `MaxContentRating` already applied there), inaccessible items drop out naturally. No extra access check needed on the id-sets.
|
||||
|
||||
**Cold/empty cache:** if the Popular cache or Trending snapshot is empty (fresh server, worker hasn't run, provider unconfigured), the id-set is empty and the preset returns no rows. This must render the empty state, not an error.
|
||||
|
||||
## Watched overlay
|
||||
|
||||
After Step 1, decorate the windowed events with per-profile watched status:
|
||||
|
||||
- One bounded lookup: `SELECT media_item_id FROM user_watch_progress WHERE user_id = $u AND profile_id = $p AND completed = true AND media_item_id = ANY($episodeContentIds)` over the window's episode/movie content_ids.
|
||||
- Watched items render dimmed + ✓. This is a per-profile overlay applied after Step 1 (like the Following filter), so it does not affect the shareability of the Step-1 candidate set.
|
||||
- Premiere / finale / new-season badges extend the existing `buildBadges` logic in `internal/api/handlers/calendar.go`.
|
||||
|
||||
## UI / UX
|
||||
|
||||
- **Selector** (`web/src/pages/Calendar.tsx`): segmented pills on desktop, collapsing to a dropdown on narrow viewports — the same responsive swap the week navigator already uses. The library picker stays beside it. Both presentations bind to one selected-preset state.
|
||||
- **Empty state**: when **Following** has no upcoming airings (new or sparse profile), show a nudge with quick switches to Popular / Trending / Everything instead of a blank week.
|
||||
- **Persistence**: remember the profile's last-chosen preset in `localStorage` (keyed by profile id) so the calendar reopens where the user left it. Server-side profile preference is a possible future enhancement.
|
||||
- **Card cues** (`web/src/components/calendar/CalendarEventCard.tsx`, `web/src/lib/upcomingEventPresentation.ts`): badges + watched overlay as above.
|
||||
|
||||
## API contract
|
||||
|
||||
- Extend `/calendar`'s `filter` parameter to accept `following | popular | trending | everything`.
|
||||
- **Keep `all | favorites | watchlist` valid** so existing `silo-android` / `silo-apple` clients keep working unchanged. `all` and `everything` are equivalent. This is purely additive.
|
||||
- Add an additive `watched: bool` field (and any new badge string) to each event item. Older clients ignore unknown fields.
|
||||
- Default when `filter` is omitted: today it is `all`. The **web client** will request `following` explicitly; the server default stays `all` to avoid changing behavior for existing API consumers. (Default-view personalization is a client choice, not a server-contract change.)
|
||||
- **Profile requirement:** the `/calendar` route already runs behind `RequireProfile`, so a profile is always present. `following` (and legacy `favorites` / `watchlist`) plus the watched overlay are per-profile and resolve against it; `popular` / `trending` / `everything` / `all` are profile-agnostic and ignore it.
|
||||
|
||||
### Multi-repo coordination (per CLAUDE.md)
|
||||
|
||||
- `silo-android` / `silo-apple`: may adopt the new presets and the watched cue as independent follow-ups. Nothing here forces a lockstep change because old filter values and response shape remain valid.
|
||||
|
||||
## Implementation surface
|
||||
|
||||
Backend:
|
||||
- `internal/catalog/calendar_repo.go` — add an id-set restriction to `CalendarFilter`; generalize `appendPersonalFilterClause` into a single `ANY($set)` path; keep library/rating clauses intact.
|
||||
- A followed-id-set resolver, `ListFollowedSeriesIDs(ctx, userID, profileID)`, placed in the package that owns the favorites/watchlist/episode join; reuse the `GetPopularItems` rollup expression.
|
||||
- `internal/api/handlers/calendar.go` — validate the expanded `filter` set; orchestrate preset → id-set resolution (Following resolver, Popular cache read, Trending snapshot read); apply the watched overlay; extend `buildBadges`.
|
||||
|
||||
Frontend:
|
||||
- `web/src/pages/Calendar.tsx` — responsive selector + preset persistence + empty state.
|
||||
- `web/src/hooks/queries/calendar.ts` — new `filter` values; `watched` field on the event type.
|
||||
- `web/src/components/calendar/CalendarEventCard.tsx` — watched overlay + badge rendering.
|
||||
- `web/src/lib/upcomingEventPresentation.ts` — badge/treatment helpers.
|
||||
|
||||
## Edge cases
|
||||
|
||||
- Cold/empty Popular cache or Trending snapshot → empty preset → empty state, never an error.
|
||||
- Profile with no follows → Following empty → empty state with preset nudges.
|
||||
- Large followed-set → bounded array today; correlated-`EXISTS` fallback noted for later.
|
||||
- Movie vs episode/season id matching (`content_id` vs `series_id`) handled by the existing `itemIDExpr` selection per branch.
|
||||
- Specials (`season_number = 0`) remain excluded.
|
||||
- Access control (`MaxContentRating`, library scoping, disabled libraries) still enforced for every preset via Step 1; viewer-agnostic id-sets cannot leak inaccessible items because of the intersection.
|
||||
- Backward-compat filter values (`all | favorites | watchlist`) still resolve through the unified id-set path with identical results.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
- **Repo** (`internal/catalog/calendar_repo_test.go`): each preset's id-set filter (following / popular / trending / everything), empty id-set → zero rows, access + rating enforcement preserved across presets, movie-by-content_id vs series-by-series_id matching, legacy favorites/watchlist parity.
|
||||
- **Handler** (`internal/api/handlers/calendar_test.go`): filter validation accepts new + legacy values; Popular/Trending cold cache degrades to empty gracefully; `watched` field populated correctly; watched lookup bounded to the window.
|
||||
- **Frontend**: selector responsive swap (pills ↔ dropdown), preset persistence across reloads, watched overlay + badge rendering, Following empty state.
|
||||
|
||||
## Future work
|
||||
|
||||
- Cross-user cache of the Step-1 windowed candidate set keyed by (window, access-signature), if load metrics justify it.
|
||||
- External enrichment of followed shows whose next season is not yet in the library.
|
||||
- Server-side persisted preset preference per profile.
|
||||
- Optional union of multiple Trending sources/windows.
|
||||
@@ -0,0 +1,208 @@
|
||||
# Trending Discover — Persistent Snapshot Design
|
||||
|
||||
Status: Approved (design)
|
||||
Date: 2026-05-29
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
The `trending_discover` home section pulls global trending (TMDB or Trakt),
|
||||
matches it to titles in the viewer's libraries, and renders the result. Today
|
||||
the external fetch + external-ID resolution are memoised in an **in-process
|
||||
map** (`editorialCandidateCache`) keyed by `source|window|fetchLimit` with a
|
||||
1-hour TTL and a `singleflight` group to collapse concurrent misses
|
||||
(`internal/sections/fetcher.go`, `loadTrendingDiscoverContentIDs`).
|
||||
|
||||
That cache has three weaknesses, two of which we explicitly want to fix:
|
||||
|
||||
1. **No resilience to upstream failure.** At TTL expiry a request still blocks
|
||||
on the TMDB/Trakt call, and a slow or down provider degrades the home page.
|
||||
There is no concept of "serve the last good list."
|
||||
2. **No observability.** Nothing records when trending last refreshed, whether
|
||||
it succeeded, or how many trending titles matched the catalog.
|
||||
3. (Secondary) It is per-process and volatile — lost on restart, and each
|
||||
server instance fetches independently.
|
||||
|
||||
## Goals
|
||||
|
||||
- Reads of the trending section **never call the upstream provider** and never
|
||||
block on it. They serve a persisted, last-good list.
|
||||
- A persisted, inspectable record with refresh history (the same "synced
|
||||
record" shape used by library collections).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Reusing the library-collection machinery. `library_collections.library_id` is
|
||||
`NOT NULL` (collections are library-scoped); trending is deliberately
|
||||
library-agnostic and appears once. Making collections library-agnostic is a
|
||||
large, separate change and is out of scope.
|
||||
- An admin UI for the snapshot. The persisted columns make this possible later,
|
||||
but it is not part of this work.
|
||||
- A shared/distributed cache (Redis). The DB is the single source of truth.
|
||||
|
||||
## Approach
|
||||
|
||||
Replace the lazy in-process cache with a **background-refreshed persistent
|
||||
snapshot**. A scheduled task refreshes the trending list on an interval; the
|
||||
read path only ever reads the persisted snapshot. This mirrors the
|
||||
collection-sync *pattern* (`CollectionSyncScheduler` + `SyncCollectionsTask`)
|
||||
without reusing its library-scoped storage.
|
||||
|
||||
The change cleanly separates **read** from **refresh**:
|
||||
|
||||
- The refresh path owns the upstream call and external-ID resolution.
|
||||
- The read path owns only a primary-key snapshot lookup plus the existing
|
||||
per-viewer access filtering.
|
||||
|
||||
## Data model
|
||||
|
||||
New migration `167_trending_discover_snapshots`. (Originally numbered 166, but
|
||||
the shared dev DB had already recorded version 166 from another branch's
|
||||
`166_trending_blend_collection_type`; the migration runner dedupes by integer
|
||||
version, so 166 was silently skipped. Renumbered to 167, the next free version.)
|
||||
|
||||
```
|
||||
trending_discover_snapshots
|
||||
source text NOT NULL -- 'tmdb' | 'trakt'
|
||||
window text NOT NULL -- 'day' | 'week' (trakt canonicalized to 'week')
|
||||
content_ids text[] NOT NULL DEFAULT '{}' -- ordered, resolved to library catalog, capped at 200
|
||||
entry_count int NOT NULL DEFAULT 0 -- raw provider entries fetched (matched-vs-trending observability)
|
||||
refreshed_at timestamptz -- last successful refresh
|
||||
last_attempt_at timestamptz
|
||||
last_status text -- 'ok' | 'empty' | 'error'
|
||||
last_error text
|
||||
PRIMARY KEY (source, window)
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- One row per canonical `(source, window)` — at most a handful of rows.
|
||||
- `content_ids` is catalog-matched but **viewer-agnostic**, exactly like the
|
||||
value the current cache holds. Per-viewer access filtering still happens at
|
||||
read time in `fetchItemsByContentIDs`, so a single row serves all viewers.
|
||||
- The list is stored at the over-fetch cap (200). The read path truncates to the
|
||||
section's `ItemLimit`, decoupling the snapshot from per-section limits (the old
|
||||
`fetchLimit` cache-key dimension goes away).
|
||||
- **Reliability invariant:** a failed refresh updates `last_attempt_at`,
|
||||
`last_status`, and `last_error` but **never clears `content_ids`**. The last
|
||||
good list keeps serving through an upstream outage.
|
||||
|
||||
The paired `.down.sql` drops the table.
|
||||
|
||||
## Components
|
||||
|
||||
All new server code lives in `internal/`.
|
||||
|
||||
### `TrendingSnapshotRepository` (new, `internal/sections/`)
|
||||
|
||||
Thin repository over the new table:
|
||||
|
||||
- `Get(ctx, source, window) (TrendingSnapshot, error)` — PK lookup.
|
||||
- `Upsert(ctx, snapshot) error` — insert/update on `(source, window)`.
|
||||
- `ListAll(ctx) ([]TrendingSnapshot, error)` — for future admin/inspection use
|
||||
and tests.
|
||||
|
||||
`Upsert` distinguishes a successful refresh (writes `content_ids`,
|
||||
`refreshed_at`, `entry_count`, `last_status='ok'|'empty'`) from a failed one
|
||||
(touches only `last_attempt_at`/`last_status='error'`/`last_error`, leaving
|
||||
`content_ids` intact).
|
||||
|
||||
### `TrendingRefresher` (new, `internal/sections/trending_refresher.go`)
|
||||
|
||||
Owns the upstream call. Holds `TMDBTrending`, `TraktTrending`, `ItemRepo`, the
|
||||
section repository (to enumerate used combos), and the snapshot repository. The
|
||||
existing `fetchTrendingDiscoverEntries` and `resolveTrendingDiscoverIDs` logic
|
||||
**moves here** from `fetcher.go`.
|
||||
|
||||
`RunOnce(ctx) (json.RawMessage, error)`:
|
||||
|
||||
1. Enumerate the distinct canonical `(source, window)` pairs from **enabled**
|
||||
`trending_discover` sections across all scopes (parse each section's
|
||||
`TrendingDiscoverParams`, apply the same source/window normalization the
|
||||
section uses, collapse Trakt to `week`).
|
||||
2. For each pair: fetch the cap-200 list from the provider, resolve external IDs
|
||||
to library content IDs, and `Upsert`.
|
||||
3. Return a JSON summary (e.g. `{combos, refreshed, empty, failed}`), mirroring
|
||||
`CollectionSyncResult`.
|
||||
|
||||
No used combos → no upstream calls (a dormant feature costs nothing). A
|
||||
configured-but-empty provider yields `last_status='empty'` (no error noise). A
|
||||
per-pair failure is recorded and does not abort the other pairs.
|
||||
|
||||
### `RefreshTrendingDiscoverTask` (new, `internal/taskmanager/tasks/`)
|
||||
|
||||
Mirrors `SyncCollectionsTask`: delegates to `TrendingRefresher.RunOnce` and
|
||||
reports progress/result data. Default trigger: hourly interval (matches the
|
||||
recipe's `DefaultCacheTTL` of 1h), plus a one-shot run at startup so the first
|
||||
snapshot lands quickly rather than after a full interval.
|
||||
|
||||
### `Fetcher` (modified, `internal/sections/fetcher.go`)
|
||||
|
||||
- `loadTrendingDiscoverContentIDs` now reads the snapshot row via the snapshot
|
||||
repository and returns its `content_ids`. It drops the upstream fetch, the
|
||||
`singleflight` group, and the `editorialCandidateCache` usage **for trending**
|
||||
(the editorial-candidate cache remains for the editorial sections that still
|
||||
use it).
|
||||
- `fetchTrendingDiscover` keeps its read-time responsibilities: access
|
||||
filtering, re-ordering to trending rank, truncation to `ItemLimit`. It no
|
||||
longer computes a `fetchLimit` for the read path.
|
||||
- The `TMDBTrending`, `TraktTrending`, and trending-only use of `ItemRepo`
|
||||
migrate out of the read path into the refresher. The fetcher gains the
|
||||
snapshot repository dependency.
|
||||
|
||||
### Wiring (`cmd/silo/main.go`, `internal/api/router.go`)
|
||||
|
||||
- Construct `TrendingSnapshotRepository` and `TrendingRefresher` (wired with the
|
||||
TMDB/Trakt fetchers, `ItemRepo`, section repo, snapshot repo).
|
||||
- Register `RefreshTrendingDiscoverTask` alongside the other task registrations.
|
||||
- Hand the snapshot repository to the section `Fetcher`; move the upstream
|
||||
fetchers from the fetcher wiring to the refresher wiring.
|
||||
|
||||
## Data flow
|
||||
|
||||
**Refresh (background, hourly + startup):**
|
||||
task → `Refresher.RunOnce` → enumerate distinct canonical `(source, window)`
|
||||
from enabled `trending_discover` sections → per pair: fetch cap-200 → resolve
|
||||
external IDs → `Upsert` snapshot.
|
||||
|
||||
**Read (per home render):**
|
||||
`fetchTrendingDiscover` → `loadTrendingDiscoverContentIDs` reads the snapshot
|
||||
row → `fetchItemsByContentIDs` applies the viewer's access filter → re-order to
|
||||
trending rank → truncate to `ItemLimit`. No upstream call. A single PK lookup on
|
||||
a ≤6-row table — negligible next to the queries `FetchAll` already runs, so no
|
||||
in-process read cache is added (it would re-muddy the single-source-of-truth and
|
||||
is premature).
|
||||
|
||||
## Edge behavior
|
||||
|
||||
- **Before first sync:** snapshot absent → section renders empty (no blocking,
|
||||
no upstream call). The startup task run closes this gap to roughly one fetch
|
||||
duration.
|
||||
- **Provider unconfigured but a section exists:** `last_status='empty'`, no
|
||||
error.
|
||||
- **Upstream failure on a pair that already has a snapshot:** prior
|
||||
`content_ids` keep serving; only the attempt/status/error columns update.
|
||||
- **Trakt `day` vs `week`:** collapse to a single `week` row (Trakt ignores the
|
||||
window).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Repository round-trip:** upsert then get; `content_ids` ordering preserved;
|
||||
success vs failure upsert paths.
|
||||
- **Refresher `RunOnce`:** fake `TMDBTrending`/`TraktTrending`, fake section
|
||||
enumerator, in-memory `ItemRepo`. Assert ordered IDs upserted; `ok`/`empty`/
|
||||
`error` transitions; and the reliability invariant — a failed refresh
|
||||
preserves prior `content_ids`.
|
||||
- **Read path:** inject nil upstream fetchers into the fetcher and assert the
|
||||
trending read still works from the snapshot (proves reads never call upstream).
|
||||
- **Canonicalization:** Trakt `day` and `week` resolve to one row.
|
||||
|
||||
## Risks / follow-ups
|
||||
|
||||
- The data captured (`refreshed_at`, `last_status`, `last_error`, `entry_count`)
|
||||
enables a future admin/inspection surface; not built here.
|
||||
- If multiple sections request very different `ItemLimit`s, all share the cap-200
|
||||
list and truncate — intended, and strictly more data than the old per-limit
|
||||
cache held.
|
||||
```
|
||||
@@ -0,0 +1,353 @@
|
||||
# Multi-Instance Sonarr/Radarr Routing for Requests (Seerr-style)
|
||||
|
||||
**Status:** Design approved, ready for implementation plan
|
||||
**Date:** 2026-06-01
|
||||
**Area:** `internal/requests`, `migrations/`, `web/src` (admin settings + request queue)
|
||||
|
||||
## Summary
|
||||
|
||||
Silo's request system today fulfills every approved request through a single
|
||||
Radarr (movies) or single Sonarr (series) instance, at whatever quality that
|
||||
one instance's profile is configured for. There is no 4K concept, no anime
|
||||
handling, and exactly one instance per kind (`request_integrations.kind` is the
|
||||
primary key).
|
||||
|
||||
This work replicates Seerr's Sonarr/Radarr management model *inside* Silo's own
|
||||
request system (no external Seerr dependency):
|
||||
|
||||
- **Many instances per kind** — N Radarr and N Sonarr servers, each with its own
|
||||
connection, root folder, quality profile, and tags.
|
||||
- **Default HD + Default 4K routing** — one HD-default and one 4K-default
|
||||
instance per kind; requests route to the matching default(s).
|
||||
- **Per-server anime overrides** (Overseerr-style) — each instance carries an
|
||||
optional anime profile / root folder / tags applied when a title is detected
|
||||
as anime.
|
||||
- **Entitlement-driven dual-quality fan-out** — a single user request can be
|
||||
fulfilled in **both** 1080p and 2160p when the requester's `MaxPlaybackQuality`
|
||||
allows 4K, or when an admin global toggle forces it for everyone.
|
||||
|
||||
The end-user request action is unchanged: quality is derived from the user's
|
||||
existing entitlement, not a per-request choice.
|
||||
|
||||
## Goals
|
||||
|
||||
- Support multiple Radarr/Sonarr instances per kind.
|
||||
- Route by quality (HD vs 4K) to per-kind default instances.
|
||||
- Fan a single request out to both 1080p and 2160p targets based on the
|
||||
requester's `MaxPlaybackQuality` entitlement, with an admin override that
|
||||
forces dual-quality regardless of role.
|
||||
- Apply per-instance anime overrides using Seerr-exact anime detection.
|
||||
- Keep the request lifecycle (quotas, approval, reconcile) coherent under
|
||||
one-to-many fulfillment.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No integration *with* a Seerr instance (an earlier idea, dropped). Silo
|
||||
remains the system of record.
|
||||
- No per-request manual server/profile picker (Overseerr's advanced UI). Routing
|
||||
is automatic from defaults + entitlement.
|
||||
- No separate 1080p/4K *availability* tracking. The pre-request presence check
|
||||
stays binary ("is this title in the Silo library at all").
|
||||
- No admin-configurable anime keyword list (the heuristic is isolated so one can
|
||||
be added later).
|
||||
- No dedicated-anime-instance flag — anime is handled by per-instance overrides
|
||||
(decided during design).
|
||||
|
||||
## Background: current flow
|
||||
|
||||
When a user requests a movie today (`internal/requests/service.go`):
|
||||
|
||||
1. **Browse/discover** via TMDB; each result carries a `RequestState`.
|
||||
2. `CreateRequest` validates: requests enabled, not already available
|
||||
(presence), not already actively requested (dedup by `tmdb_id`), computes the
|
||||
requester's `EffectivePolicy` (quota, blocked, auto-approve), enforces quota.
|
||||
3. Auto-approve users with a configured integration → row created `approved`,
|
||||
else `pending` for the admin queue.
|
||||
4. `submitApprovedRequest` → `integrationForMediaType(mediaType)` finds the
|
||||
single instance by kind → `radarr.Client.SubmitMovie` POSTs to Radarr using
|
||||
that instance's root folder / quality profile / tags → request goes `queued`.
|
||||
5. `ReconcileRequests` polls the instance: `queued → downloading → completed`
|
||||
(or `failed`).
|
||||
|
||||
The single insertion point that changes is step 4's instance selection.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Data model
|
||||
|
||||
**`request_integrations` — reworked from one-row-per-kind to many instances:**
|
||||
|
||||
| column | purpose |
|
||||
|---|---|
|
||||
| `id` (text PK, via `idgen`) | replaces `kind` as PK |
|
||||
| `kind` (text, `radarr`\|`sonarr`) | now a plain column, keeps its CHECK |
|
||||
| `name` (text) | admin-facing label |
|
||||
| `enabled` (bool) | |
|
||||
| `base_url`, `api_key_ref` | connection; API key stays a Fernet secret ref, never plaintext |
|
||||
| `root_folder`, `quality_profile_id`, `tags[]` | standard (non-anime) defaults |
|
||||
| `is_4k` (bool) | marks this as a 4K server |
|
||||
| `is_default` (bool) | the HD default for its kind |
|
||||
| `is_default_4k` (bool) | the 4K default for its kind |
|
||||
| `anime_enabled` (bool) | apply anime overrides on this instance |
|
||||
| `anime_quality_profile_id`, `anime_root_folder`, `anime_tags[]` | anime overrides |
|
||||
| `options` (jsonb) | kind-specific extras (Radarr min-availability, Sonarr season-folders/series-type) |
|
||||
| `last_check_*`, `created_at`, `updated_at` | unchanged |
|
||||
|
||||
**Invariants** (partial unique indexes + service validation):
|
||||
|
||||
- At most one `is_default` per kind; at most one `is_default_4k` per kind.
|
||||
- `is_default ⇒ NOT is_4k` and `is_default_4k ⇒ is_4k`. The HD default is an HD
|
||||
server; the 4K default is a 4K server. HD and 4K defaults are therefore always
|
||||
distinct instances.
|
||||
|
||||
**New `media_request_targets`:**
|
||||
|
||||
| column | purpose |
|
||||
|---|---|
|
||||
| `id` (bigint identity PK) | |
|
||||
| `request_id` (text → `media_requests` ON DELETE CASCADE) | |
|
||||
| `integration_id` (text → `request_integrations` ON DELETE SET NULL) | |
|
||||
| `integration_kind` (text snapshot) | survives instance deletion for history |
|
||||
| `quality` (text, `1080p`\|`2160p`) | reuses `internal/access` presets |
|
||||
| `is_anime` (bool) | which profile set was applied |
|
||||
| `external_id`, `external_status`, `status`, `last_error` | per-target Radarr/Sonarr lifecycle |
|
||||
| `created_at`, `updated_at` | |
|
||||
| UNIQUE `(request_id, quality)` | one target per quality per request |
|
||||
|
||||
**`media_requests` changes:** the per-fulfillment columns
|
||||
(`integration_kind`, `external_id`, `external_status`) move out to
|
||||
`media_request_targets`. Add `is_anime` (bool). The request's `status` becomes an
|
||||
**aggregate** over its targets (see §3). `idx_media_requests_active_tmdb` stays —
|
||||
dedup is still one active request per title regardless of quality, and a
|
||||
dual-quality request counts as **one** row against the user's quota.
|
||||
|
||||
### 2. Routing engine
|
||||
|
||||
A pure function `routeTargets(req, entitlement, settings, instances) -> []plannedTarget`
|
||||
replaces `integrationForMediaType` as the selection point in
|
||||
`submitApprovedRequest`.
|
||||
|
||||
**Inputs:** the request (incl. detected `is_anime`), the requester's
|
||||
`MaxPlaybackQuality`, the global `force_dual_quality` setting, the configured
|
||||
instances for the kind.
|
||||
|
||||
**Algorithm:**
|
||||
|
||||
1. **Kind** = `movie → radarr`, `series → sonarr`.
|
||||
2. **Desired qualities:**
|
||||
- `1080p` is always desired (baseline for everyone).
|
||||
- Add `2160p` if `access.QualityAllowed("2160p", user.MaxPlaybackQuality)`
|
||||
**OR** `force_dual_quality` is on (the toggle overrides role).
|
||||
3. **Pick the default instance per desired quality:** `1080p → is_default`,
|
||||
`2160p → is_default_4k` (for that kind).
|
||||
4. **Profile selection per chosen instance:** if `req.is_anime` **and** the
|
||||
instance has `anime_enabled` → use its anime profile / root folder / tags
|
||||
(and, for Sonarr, `seriesType: anime`); otherwise the standard set.
|
||||
5. Emit one `plannedTarget{instance, quality, isAnime, profile, rootFolder, tags}`
|
||||
per resolved quality.
|
||||
|
||||
**Edge cases:**
|
||||
|
||||
- A desired quality with **no default instance assigned** is silently skipped
|
||||
(cannot route 4K without a 4K default — matches "when both are assigned").
|
||||
- **Zero** resolved targets (e.g. no HD default configured): the request stays
|
||||
`approved` but unfulfilled, with `last_error` = "no Radarr/Sonarr instance
|
||||
configured for this quality" so it surfaces in the admin queue.
|
||||
- No dedup needed: the `is_default`/`is_default_4k` invariants guarantee HD and
|
||||
4K defaults are distinct instances.
|
||||
|
||||
### 3. Fulfillment & reconcile lifecycle
|
||||
|
||||
The adapter interfaces (`SubmitMovie`/`SubmitSeries`,
|
||||
`CheckMovieStatus`/`CheckSeriesStatus`) are **unchanged** — they already take
|
||||
`(req, integration)`. We call them **once per target**, passing a *resolved*
|
||||
`Integration` whose `RootFolder`/`QualityProfileID`/`Tags` are filled from the
|
||||
standard or anime block. Radarr/Sonarr's notion of "4K" is just that instance's
|
||||
quality profile + root folder, so adapters stay quality-agnostic. (The Sonarr
|
||||
adapter additionally sets `seriesType: anime` for anime targets.)
|
||||
|
||||
**On approval (`submitApprovedRequest`):**
|
||||
|
||||
1. `routeTargets(...)` → planned targets.
|
||||
2. For each: insert a `media_request_targets` row, then call the matching
|
||||
adapter with the resolved instance + decrypted key.
|
||||
3. On success → target `queued` with its `external_id`/`external_status`; on
|
||||
error → target `failed` + `last_error`.
|
||||
4. **Recompute the request aggregate** in the same transaction as the target
|
||||
writes.
|
||||
|
||||
**Reconcile loop (`ReconcileRequests`):** for each candidate request, iterate its
|
||||
**non-terminal targets**, load each target's instance by `integration_id`, call
|
||||
the status adapter, update the target, then recompute the aggregate. Targets
|
||||
reconcile independently.
|
||||
|
||||
**Aggregate recompute (single source of truth):**
|
||||
|
||||
- all targets `completed` → request `completed` (+ `completed_at`)
|
||||
- any target `downloading` → `downloading`; else any `queued` → `queued`
|
||||
- all targets `failed` → `outcome = failed`; **some** failed while others active
|
||||
→ request stays active, `last_error` set from the failed target(s)
|
||||
|
||||
**Retry** is target-aware: an admin retry re-submits only the `failed` targets
|
||||
(re-routing if instance config changed), leaving healthy targets alone.
|
||||
|
||||
### 4. Anime detection (Seerr-exact)
|
||||
|
||||
Detection matches upstream Seerr exactly — a single TMDB keyword id, no genre or
|
||||
language fallback:
|
||||
|
||||
- `server/api/themoviedb/constants.ts`: `ANIME_KEYWORD_ID = 210024`
|
||||
- `server/entity/MediaRequest.ts`: anime ⇔ `keywords.results` contains that id.
|
||||
|
||||
Implementation:
|
||||
|
||||
- A named constant `animeKeywordID = 210024`.
|
||||
- `detectAnime(detail) bool` → true iff the TMDB keyword **ids** contain
|
||||
`210024`. Matching by id (not the name string) avoids false positives from
|
||||
neighboring keywords like "based on anime".
|
||||
- Detected once at `CreateRequest` time and stored on `media_requests.is_anime`,
|
||||
so routing, reconcile, and retries stay consistent.
|
||||
- Applies to both movies and series; ignored if no instance for that kind has
|
||||
`anime_enabled`.
|
||||
|
||||
**Supporting TMDB-client change:** extend `tmdb.MediaDetail` to carry keyword
|
||||
**ids** (add `append_to_response=keywords` to the detail fetch). `GetMediaDetail`
|
||||
is already called in the request path, so no extra round-trip beyond the append.
|
||||
(`OriginalLanguage` is not needed under the parity-only decision but may be
|
||||
mapped opportunistically since it is already in the response.)
|
||||
|
||||
The heuristic is isolated in one function so an admin-configurable keyword list
|
||||
can be added later without touching routing.
|
||||
|
||||
### 5. Admin UI
|
||||
|
||||
`web/src/pages/admin-settings/IntegrationsSettings.tsx` and the setup wizard
|
||||
`web/src/pages/setup-wizard/steps/IntegrationsStep.tsx` move from a single
|
||||
Radarr/Sonarr form to an **instance-list manager** per kind:
|
||||
|
||||
**Per kind — list of instance cards** (add / edit / delete):
|
||||
|
||||
- Connection: name, base URL, API key (write-only; shows "configured" once set,
|
||||
never echoes the secret), **Test connection** → `LoadIntegrationOptions`
|
||||
populates root-folder and quality-profile **dropdowns**.
|
||||
- Standard block: root folder, quality profile, tags.
|
||||
- Quality role: `is_4k` switch, **Default (HD)** toggle, **Default 4K** toggle.
|
||||
The form enforces invariants client-side (no Default-HD on a 4K server;
|
||||
selecting a new default clears the prior one visually); server is source of
|
||||
truth.
|
||||
- Anime block (collapsible, `anime_enabled`): anime quality profile, anime root
|
||||
folder, anime tags — dropdowns from the same test-connection options.
|
||||
|
||||
**Request settings panel** gains the **"Always fulfill in both 1080p and 4K"**
|
||||
toggle (`force_dual_quality`) with helper text: applies to all requests when both
|
||||
a Default HD and Default 4K instance exist, regardless of user role.
|
||||
|
||||
**`web/src/pages/AdminRequests.tsx`** (queue): each request row expands to show
|
||||
its **targets** — quality badge (1080p/2160p), instance name, per-target status,
|
||||
and per-target **retry** on failed ones. The aggregate status stays the headline.
|
||||
|
||||
**Setup wizard** stays minimal: add one Radarr + one Sonarr, auto-marked Default
|
||||
HD. Advanced multi-instance/4K/anime config lives in the full settings page.
|
||||
|
||||
### 6. API contract & multi-repo coordination
|
||||
|
||||
**End-user request flow is unchanged** — quality is derived from
|
||||
`MaxPlaybackQuality`, so the request action and payload are identical (no 4K
|
||||
toggle, no new permission).
|
||||
|
||||
**`Request` response model** gains `is_anime` and a `targets[]` array
|
||||
(`quality`, `instance_name`, `status`, `external_status`, `last_error`); the
|
||||
top-level `integration_kind`/`external_id`/`external_status` are removed in favor
|
||||
of the aggregate `status` + `targets`.
|
||||
|
||||
- Per `CLAUDE.md`'s multi-repo rule: end-user `silo-android` / `silo-apple`
|
||||
clients show a single aggregate status, so they likely need **no change** —
|
||||
but this is flagged as explicit Apple/Android follow-up to verify the request
|
||||
model tolerates the removed fields and ignores/parses the new `targets`.
|
||||
|
||||
**Admin endpoints** change from "upsert by kind" to instance CRUD:
|
||||
|
||||
- `GET/POST/PUT/DELETE` request integrations **by `id`**; `LoadIntegrationOptions`
|
||||
(test-connection) stays, keyed by instance.
|
||||
- Default-toggle handled server-side transactionally (a new HD/4K default clears
|
||||
the prior one for that kind).
|
||||
- `request_settings` gains `force_dual_quality`.
|
||||
- Admin-web-only; no mobile client surface.
|
||||
|
||||
### 7. Migration & backfill
|
||||
|
||||
Next migration number: **169** (`169_request_multi_instance.{up,down}.sql`).
|
||||
|
||||
**Up:**
|
||||
|
||||
1. `ALTER request_integrations`: add `id`, `name`, `is_4k`, `is_default`,
|
||||
`is_default_4k`, `anime_enabled`, `anime_quality_profile_id`,
|
||||
`anime_root_folder`, `anime_tags[]`. Backfill existing rows: `id = gen`,
|
||||
`name = initcap(kind)`, `is_default = enabled` (the lone existing instance
|
||||
becomes that kind's HD default), everything else default/false. Swap the PK
|
||||
from `kind` to `id`; keep `kind` as a plain column with its CHECK.
|
||||
2. Partial unique indexes: one `is_default` per kind, one `is_default_4k` per
|
||||
kind.
|
||||
3. `CREATE media_request_targets` (per §1).
|
||||
4. Backfill targets: for any `media_requests` row with an `external_id`, insert a
|
||||
target `(quality='1080p', is_anime=false, integration_id = that kind's
|
||||
instance, integration_kind, external_id, external_status, status = request's
|
||||
current status)`. Rows never submitted get no target yet.
|
||||
5. `ALTER media_requests`: add `is_anime bool DEFAULT false`; drop
|
||||
`integration_kind`, `external_id`, `external_status`.
|
||||
6. `idx_media_requests_active_tmdb` stays unchanged.
|
||||
|
||||
**Down:** re-add the three columns to `media_requests` and copy back the **1080p**
|
||||
target's external fields; drop `media_request_targets`; drop `is_anime`; collapse
|
||||
`request_integrations` to `kind`-PK by keeping each kind's `is_default` instance
|
||||
and discarding extras; drop the new columns/indexes.
|
||||
|
||||
**Caveat (explicit):** the down migration is **lossy** — rollback discards any
|
||||
additional instances (beyond one default per kind) and any 4K/anime target
|
||||
history. This is the unavoidable cost of collapsing a one-to-many back to
|
||||
one-to-one and is acceptable for a down migration.
|
||||
|
||||
**Go-side data access** (implementation, not SQL): `Integration` struct +
|
||||
`scanIntegration` gain the new fields; the `Store` interface gains target CRUD +
|
||||
`ListInstances(kind)`; `scanRequest` drops the external fields; `UpsertIntegration`
|
||||
becomes id-based CRUD with default-toggle handling in-transaction.
|
||||
|
||||
### 8. Testing
|
||||
|
||||
- **`routeTargets` — table-driven (core of the feature):** matrix of
|
||||
{1080p vs 2160p entitlement} × {`force_dual_quality` on/off} × {which defaults
|
||||
exist} × {anime vs not} → expected target set.
|
||||
- **`detectAnime`:** keyword `210024` present/absent; no name-substring false
|
||||
positives.
|
||||
- **Aggregate recompute:** all-complete, partial-failure (request stays active +
|
||||
`last_error`), all-failed (`outcome=failed`), mixed downloading/queued.
|
||||
- **Repository:** target CRUD; single-default-per-kind invariant rejects a second
|
||||
default; migration backfill yields exactly one 1080p target per previously
|
||||
submitted request.
|
||||
- **Adapter:** Sonarr `SubmitSeries` sets `seriesType=anime` only for anime
|
||||
targets on anime-enabled instances.
|
||||
- **Service:** extend `service_test.go` fakes — a 4K-entitled user's single
|
||||
request fans out to two adapter submissions with the right resolved instances;
|
||||
target-scoped retry re-submits only the failed target.
|
||||
- **Frontend:** instance-form default-invariant enforcement; queue renders
|
||||
per-target rows + retry.
|
||||
|
||||
## Risks & open considerations
|
||||
|
||||
- **Reconcile cost** scales with targets, not requests (≤2× today). Bounded and
|
||||
acceptable; the reconcile loop already batches.
|
||||
- **Anime keyword imperfection** is inherited from Seerr by design (parity
|
||||
choice). Mitigated by isolating `detectAnime` for future tuning.
|
||||
- **Client model drift**: removing top-level external fields from `Request` must
|
||||
be verified against `silo-android` / `silo-apple` deserialization before
|
||||
release (flagged in §6).
|
||||
|
||||
## References
|
||||
|
||||
- Seerr upstream: https://github.com/seerr-team/seerr
|
||||
(`server/api/themoviedb/constants.ts`, `server/entity/MediaRequest.ts`)
|
||||
- Overseerr anime discussions: #2876 (Sonarr for anime only), #3777 (configurable
|
||||
series type)
|
||||
- Existing code: `internal/requests/{service,repository,types,store}.go`,
|
||||
`internal/requests/{radarr,sonarr,arrclient}`, `internal/access/quality.go`,
|
||||
`migrations/139_media_requests.up.sql`
|
||||
@@ -0,0 +1,222 @@
|
||||
# Autoscan: Polling Radarr/Sonarr to Trigger Targeted Library Scans
|
||||
|
||||
**Status:** Design approved, ready for implementation plan
|
||||
**Date:** 2026-06-02
|
||||
**Area:** new `internal/autoscan`, `internal/taskmanager/tasks`, `migrations/`, `internal/api`, `web/src/pages/AdminRequests.tsx`
|
||||
|
||||
## Summary
|
||||
|
||||
Silo currently learns about new media via scheduled/manual library scans. When
|
||||
Radarr/Sonarr import a download, the new file only appears in Silo on the next
|
||||
scan. This adds an **autoscan poller**: a periodic background task that polls the
|
||||
configured Radarr/Sonarr instances for newly-imported files and enqueues
|
||||
**targeted folder scans** into Silo's existing scan queue, so requested (and
|
||||
non-requested) content shows up promptly.
|
||||
|
||||
The design follows the established autoscan pattern — poll arr import history,
|
||||
translate the imported paths, and trigger targeted scans — but kept deliberately
|
||||
simple. That pattern is normally built to orchestrate scans across many separate
|
||||
media-server boxes, which needs a bounded fan-out guard and a retry queue. Silo
|
||||
is a single service scanning *itself*, so the cross-node fan-out machinery is
|
||||
unnecessary — imported paths become folder jobs in Silo's own worker-bounded
|
||||
`scanqueue`.
|
||||
|
||||
## Goals
|
||||
|
||||
- Poll autoscan-enabled Radarr/Sonarr instances on an interval for import events.
|
||||
- Translate arr import paths into Silo media folders and enqueue targeted scans.
|
||||
- Reuse the Radarr/Sonarr instances already configured for requests
|
||||
(`request_integrations`) — no duplicate URLs/keys.
|
||||
- Coalesce bursts (a season's episodes) into minimal redundant scans.
|
||||
- Admin-configurable: global enable, interval, debounce, per-instance toggle,
|
||||
optional path rewrites, on-demand "poll now".
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No cross-node scan fan-out or bounded-concurrency semaphore (single service;
|
||||
the `scanqueue` workers are the bound).
|
||||
- No separate retry queue (a failed source poll simply does not advance its
|
||||
high-water mark, so the next cycle retries the window).
|
||||
- No arr webhook/"Connect" push integration (polling only, as requested; webhook
|
||||
push is a possible future add-on).
|
||||
- No separate autoscan source credential store — sources are existing
|
||||
`request_integrations`.
|
||||
|
||||
## Background
|
||||
|
||||
- **`internal/scantrigger`** already maps a filesystem path to a Silo
|
||||
`MediaFolder` and scan path: `Resolver.Resolve(Request{Path}) -> Target{Folder,
|
||||
Mode, Path}` (via `LongestMatchingRoot` / `MatchFolderForPath`).
|
||||
- **`internal/scanqueue`** executes scans through a worker-bounded queue:
|
||||
`Queuer.EnqueueScan(ctx, folderID, mode, path, trigger)` /
|
||||
`EnqueueScans(targets)`.
|
||||
- **`internal/requests/arrclient`** is a thin Radarr/Sonarr HTTP client
|
||||
(`New(baseURL, apiKey, httpClient)`, `GetJSON`, `DoJSON`) — reused for the new
|
||||
history call.
|
||||
- **`request_integrations`** holds each instance's `id`, `kind`, `base_url`, and
|
||||
Fernet-encrypted `api_key_ref`, resolved via the same secret resolver the
|
||||
request service uses.
|
||||
- The **task manager** runs periodic tasks via the `taskmanager.Task` interface
|
||||
with interval `DefaultTriggers()` (e.g. `ReconcileRequestsTask`).
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Data model & config
|
||||
|
||||
A new `internal/autoscan` package owns its own schema; `request_integrations`
|
||||
stays focused on fulfillment.
|
||||
|
||||
**`autoscan_settings`** (singleton, mirrors `request_settings`):
|
||||
|
||||
| column | type / default | purpose |
|
||||
|---|---|---|
|
||||
| `id` | bool PK default true, singleton CHECK | |
|
||||
| `enabled` | bool, default false | master on/off |
|
||||
| `poll_interval_minutes` | int, default 10 | poll task interval |
|
||||
| `debounce_seconds` | int, default 60 | folder re-scan suppression window |
|
||||
| `created_at`, `updated_at` | timestamptz | |
|
||||
|
||||
**`autoscan_sources`** (one row per autoscan-enabled instance):
|
||||
|
||||
| column | type | purpose |
|
||||
|---|---|---|
|
||||
| `integration_id` | text PK → `request_integrations(id)` ON DELETE CASCADE | the source instance |
|
||||
| `enabled` | bool, default false | per-instance autoscan toggle |
|
||||
| `path_rewrites` | jsonb, default `[]` | `[{from, to}]` prefix overrides |
|
||||
| `last_poll_at` | timestamptz, null | `history/since` high-water mark |
|
||||
| `created_at`, `updated_at` | timestamptz | |
|
||||
|
||||
**Reused, not duplicated:** `base_url` / `api_key_ref` come from
|
||||
`request_integrations`. Debounce/suppression state lives in **Redis**
|
||||
(`autoscan:scanned:{folder_id}`, TTL = `debounce_seconds`), not Postgres.
|
||||
|
||||
### 2. Poll cycle
|
||||
|
||||
`autoscan.Service.PollOnce(ctx)` (invoked by the periodic task):
|
||||
|
||||
1. Load `autoscan_settings`; if `!enabled`, return immediately (no-op).
|
||||
2. Load `autoscan_sources WHERE enabled`, joined to `request_integrations` for
|
||||
`kind` / `base_url` / `api_key_ref`.
|
||||
3. For each source (sequential — a handful of instances):
|
||||
1. Decrypt the API key via the secret resolver.
|
||||
2. Record `cycleStart := now`.
|
||||
3. `arrclient.GetJSON("/api/v3/history/since?date={last_poll_at}&eventType=downloadFolderImported")`
|
||||
— both Radarr and Sonarr expose this. `last_poll_at` null → a sane floor
|
||||
(do not scan all history on first enable; floor at `cycleStart`).
|
||||
4. Extract the imported **file path** per event (Radarr: movie file path;
|
||||
Sonarr: episode file `outputPath`); take its **parent directory**.
|
||||
5. On success, advance that source's `last_poll_at = cycleStart`. On failure
|
||||
(arr unreachable, decode error), log and **do not** advance — next cycle
|
||||
retries the window.
|
||||
4. For each extracted directory: apply path rewrites (§3), resolve via
|
||||
`scantrigger.Resolver` → `Target{Folder, Mode, Path}`. Unresolvable paths
|
||||
(outside Silo's media folders) are logged and skipped.
|
||||
5. **Dedupe to unique folders** for this cycle; for each, check the Redis
|
||||
suppression key — if absent, `EnqueueScan(..., trigger="autoscan")` and set
|
||||
the key with TTL `debounce_seconds`; if present, skip (already scanned this
|
||||
window).
|
||||
|
||||
**Debounce rationale:** because the poller batches a whole interval of imports
|
||||
per `history/since` call, within-cycle folder dedup does most of the work. The
|
||||
Redis suppression key only prevents re-enqueuing the same folder on back-to-back
|
||||
cycles. Silo's scanner is idempotent, so a season importing across two cycles
|
||||
simply gets a second (cheap) scan that picks up the rest. This is simpler than a
|
||||
"collect-expired" debounce plus a retry queue, which the polling + idempotent
|
||||
re-scan model makes unnecessary.
|
||||
|
||||
### 3. Path resolution & scan enqueue
|
||||
|
||||
A small pipeline in front of the existing scan plumbing — no new scan logic:
|
||||
|
||||
1. **Rewrite** (per source, optional): apply the first matching `path_rewrites`
|
||||
entry as a prefix replacement (e.g. `/data/media -> /mnt/media`). No rewrites
|
||||
→ path used as-is (shared-mount common case).
|
||||
2. **Resolve** via `scantrigger.Resolver.Resolve(Request{Path: rewrittenDir})`
|
||||
→ `Target{Folder, Mode, Path}`. No matching media folder → log + skip.
|
||||
3. **Enqueue** deduped targets via `scanqueue` `EnqueueScans` with
|
||||
`trigger="autoscan"` (attributable in scan history, flows through the same
|
||||
worker-bounded queue).
|
||||
|
||||
### 4. Scheduling & concurrency
|
||||
|
||||
- **`internal/taskmanager/tasks/autoscan_poll.go`** implements `taskmanager.Task`
|
||||
(`Key: "autoscan_poll"`, name/description/category, `DefaultTriggers` = interval
|
||||
trigger seeded from `poll_interval_minutes`). Its run body calls
|
||||
`autoscan.Service.PollOnce(ctx)`.
|
||||
- **Disabled = cheap no-op:** the task stays registered; `PollOnce` returns early
|
||||
when `enabled` is false. Toggling autoscan never (de)registers tasks.
|
||||
- **Interval changes:** updating `poll_interval_minutes` reconfigures the task's
|
||||
interval trigger via the task manager's existing trigger-config path; if a live
|
||||
update is unavailable it applies on next start.
|
||||
- **No overlap, no semaphore:** the task manager serializes a task's executions;
|
||||
sources are polled sequentially; scan execution is bounded by `scanqueue`. The
|
||||
single-service simplification: no bounded-fan-out semaphore is needed.
|
||||
- **Primary/integrated server only** (same as the request reconcile task), not on
|
||||
standalone transcode/proxy worker nodes.
|
||||
|
||||
### 5. Admin API & UI
|
||||
|
||||
**API** (admin-gated, `/api/v1/admin/autoscan`, mirroring the request-settings
|
||||
handlers; never echoes API keys):
|
||||
|
||||
- `GET/PUT /autoscan/settings` — `{enabled, poll_interval_minutes, debounce_seconds}`
|
||||
- `GET /autoscan/sources` — per autoscan-eligible instance (joined from
|
||||
`request_integrations`): `{integration_id, name, kind, enabled, path_rewrites,
|
||||
last_poll_at}`
|
||||
- `PUT /autoscan/sources/{integration_id}` — set `{enabled, path_rewrites}`
|
||||
- `POST /autoscan/trigger` — run `PollOnce` once on demand; tight per-admin rate
|
||||
limit so it cannot hammer the arr instances
|
||||
- `GET /autoscan/status` — `{enabled, per-source last_poll_at, last scan summary}`
|
||||
|
||||
**UI:** a new **"Autoscan" tab** in `web/src/pages/AdminRequests.tsx` (alongside
|
||||
Queue / Settings / Integrations / User Overrides):
|
||||
|
||||
- Global controls: enable switch, poll interval, debounce window.
|
||||
- Per-source list — one row per Radarr/Sonarr instance with an autoscan toggle,
|
||||
an optional `from → to` path-rewrite editor, and a read-only "last polled".
|
||||
- A **"Poll now"** button → `POST /autoscan/trigger`.
|
||||
|
||||
Rate limits and admin-auth follow Silo's existing request-admin conventions.
|
||||
|
||||
### 6. Testing
|
||||
|
||||
- **Path rewrite:** prefix replacement, first-match ordering, no-match
|
||||
passthrough, trailing-slash handling.
|
||||
- **arr history client:** table-driven `httptest` fixtures for both Radarr and
|
||||
Sonarr import-event JSON shapes; asserts the correct file path is extracted and
|
||||
its parent folder taken.
|
||||
- **Dedupe + suppression:** many paths in one folder → one target; folder with a
|
||||
live suppression key skipped; absent/expired key enqueues.
|
||||
- **`PollOnce` (integration seam):** fakes for source store, arr history client,
|
||||
`scantrigger.Resolver`, `scanqueue.Queuer`, Redis — asserts deduped folder
|
||||
scans enqueued with `trigger="autoscan"`, `last_poll_at` advanced on success,
|
||||
**not** advanced on per-source failure, clean no-op when disabled.
|
||||
- **Unresolvable path:** logged + skipped, never enqueued or fatal.
|
||||
- **Repository / migration:** `autoscan_settings` singleton + `autoscan_sources`
|
||||
CRUD and the `request_integrations` join; migration applies + rolls back
|
||||
(sidecar table, `ON DELETE CASCADE`).
|
||||
- **Frontend:** Autoscan tab renders, per-source toggle / path-rewrite editor
|
||||
round-trips, "Poll now" calls the trigger endpoint.
|
||||
|
||||
## Risks & open considerations
|
||||
|
||||
- **First-enable floor:** `last_poll_at` null must floor at "now" (not epoch) so
|
||||
enabling autoscan doesn't enqueue a scan for every historical import.
|
||||
- **history/since shape differences:** Radarr vs Sonarr import-event payloads
|
||||
differ; the path extraction is per-kind and covered by fixtures.
|
||||
- **Path rewrites vs scantrigger roots:** if neither rewrites nor Silo's media
|
||||
folders match the arr path, the import is silently skipped (logged) — this is
|
||||
the expected "content outside Silo's libraries" behavior, but a misconfigured
|
||||
rewrite will look like "autoscan does nothing"; the `last_poll_at` + status
|
||||
endpoint and skip logs are the debugging surface.
|
||||
- **Idempotent re-scan:** relies on Silo's scanner being safe to run repeatedly
|
||||
on a folder (it is — re-scan finds new files).
|
||||
|
||||
## References
|
||||
|
||||
- Existing Silo plumbing reused by this design: `internal/scantrigger/scantrigger.go`,
|
||||
`internal/scanqueue/`, `internal/requests/arrclient/`,
|
||||
`internal/taskmanager/tasks/reconcile_requests.go`
|
||||
- Prior art: the general autoscan pattern (poll arr import history → translate
|
||||
paths → trigger targeted scans). Silo's variant drops the cross-node fan-out
|
||||
guard and retry queue, which only a multi-box orchestrator needs.
|
||||
@@ -0,0 +1,262 @@
|
||||
# Autoscan as a Pluggable Scan-Source Category — Design Spec
|
||||
|
||||
**Date:** 2026-06-02
|
||||
**Status:** Approved design, pre-implementation
|
||||
**Goal:** Turn autoscan from a Requests-coupled, arr-only feature into a standalone **Autoscan** category whose change-detection providers are out-of-process Silo plugins, with Sonarr/Radarr as the first provider.
|
||||
|
||||
**Architecture (one sentence):** Silo's host keeps a thin, provider-agnostic engine that resolves changed paths to library folders and enqueues rescans; each *way of noticing change* (arr, later inotify/ceph) is an installable plugin that Silo pulls on a timer through a new additive `scan_source.v1` capability.
|
||||
|
||||
**Tech stack:** Go (silo-server host + plugins), `silo-plugin-sdk` (protobuf capability contract, consumed as a tagged module), existing `internal/plugins` + `internal/pluginhost` runtime, existing `scantrigger`/`scanqueue` scan pipeline, React/TS admin UI.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## 1. Background & motivation
|
||||
|
||||
The current autoscan implementation (delivered in the now-closed PR #43, still on local `main`) is in-process Go that:
|
||||
|
||||
- polls Sonarr/Radarr `/api/v3/history/since` on a host timer,
|
||||
- rewrites arr-side paths to Silo paths, dedups to parent dirs,
|
||||
- resolves each to a `MediaFolder` and enqueues a targeted scan,
|
||||
- is configured against `request_integrations` rows (hard FK, `ON DELETE CASCADE`), living as a tab inside the **Requests** admin page.
|
||||
|
||||
Two structural problems:
|
||||
|
||||
1. **It is arr-only and not extensible.** Adding inotify, Ceph notifications, or any other change source means more bespoke host code.
|
||||
2. **It is subordinate to Requests.** A source cannot exist without a Requests integration row; deleting the row destroys the source.
|
||||
|
||||
Silo already ships a full out-of-process plugin runtime (`internal/plugins`, `internal/pluginhost`): manifests, an installer, typed capability descriptors (`metadata_provider.v1`, `request_router.v1`, `media_analyzer.v1`, `scheduled_task.v1`, `event_consumer.v1`, …), a host-driven scheduled-task mechanism, and a rate-limited plugin→host event path. This design uses that runtime to make change-detection pluggable.
|
||||
|
||||
## 2. Goals / non-goals
|
||||
|
||||
**Goals**
|
||||
|
||||
- A new **Autoscan** admin category, independent of Requests.
|
||||
- A new additive `scan_source.v1` plugin capability — **breaks no existing plugin**.
|
||||
- A **client-pull** contract: Silo asks each source "what changed since `<marker>`?" on a timer.
|
||||
- Sonarr/Radarr re-implemented as the first scan-source plugin; the closed PR's arr logic relocates into it.
|
||||
- A provider-agnostic host engine reusable by every future scan source.
|
||||
- Optional reuse of Requests' arr connection details, *or* standalone connection entry.
|
||||
|
||||
**Non-goals (this design)**
|
||||
|
||||
- inotify / Ceph / other providers (the contract must *accommodate* them; we do not build them).
|
||||
- A push (plugin-initiated) delivery mode. Pull only; push sources buffer internally and answer on pull (see §6).
|
||||
- Refactoring the Requests subsystem itself.
|
||||
|
||||
## 3. Architecture overview
|
||||
|
||||
```
|
||||
┌──────────────────────────── Silo (host) ────────────────────────────┐
|
||||
│ Autoscan category (new; not under Requests) │
|
||||
│ • Sources: which scan_source plugin instance, on/off, interval │
|
||||
│ • Connections: reuse a Requests arr server, or enter own │
|
||||
│ • Per-source state: opaque marker, last-run status │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ AutoscanStore — generic engine (provider-agnostic) │
|
||||
│ per tick, per enabled source: │
|
||||
│ call plugin PollChanges(marker) │
|
||||
│ → resolve each Silo-native path → MediaFolder │
|
||||
│ → suppress duplicates → enqueue scan (existing scanner) │
|
||||
│ → store returned next_marker │
|
||||
│ │ scan_source.v1 (pull) │
|
||||
└──────────────────────────────┼───────────────────────────────────────┘
|
||||
▼
|
||||
┌──────── Sonarr/Radarr scan-source plugin ────────┐
|
||||
│ (new installable repo, like silo-plugin-tmdb) │
|
||||
│ receives resolved arr address + key │
|
||||
│ polls /history (imports + renames) │
|
||||
│ returns RAW source paths + next marker │
|
||||
│ (host applies the source's rewrites) │
|
||||
└──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Five components**
|
||||
|
||||
1. **`scan_source.v1`** (in `silo-plugin-sdk`) — the pull contract. §5.
|
||||
2. **AutoscanStore** (host) — generic resolve→suppress→enqueue→marker engine. §7.
|
||||
3. **Connection resolution** (host) — reference-or-inline → concrete credentials. §8.
|
||||
4. **Autoscan category + data model** (host) — sources, connections, decoupled from Requests. §9.
|
||||
5. **arr scan-source plugin** (new repo) — first provider. §10.
|
||||
|
||||
## 4. Responsibility split (the dividing line)
|
||||
|
||||
| Concern | Owner | Rationale |
|
||||
|---|---|---|
|
||||
| Talk to arr; poll `/history`; imports + renames; bounded window; first-run | **Plugin** | Provider-specific I/O. |
|
||||
| **Path rewrites** (per-source config + application) | **Host** | _(Revised post-merge: the SDK landed host-owned rewrites.)_ The plugin returns RAW source-namespace paths in `source_paths`; the host applies each source's rewrite rules (longest-`From`-first) before resolving. Keeps every provider dumb and rewrite behavior uniform. Rewrites are edited per-source in the Autoscan UI and stored in `autoscan_sources.path_rewrites`. |
|
||||
| Resolve Silo-native path → `MediaFolder`; dedupe; **enqueue rescan** | **Host** | Touches Silo internals (scan queue, DB, library map) that plugins cannot and must not reach. |
|
||||
| Opaque per-source marker; poll interval; enabled; last-run status | **Host** | Scheduling and bookkeeping are the engine's job. |
|
||||
| Connection selection (reuse-from-Requests vs own) and credential resolution | **Host** | Only the host knows about Requests; the plugin receives resolved creds. |
|
||||
|
||||
The plugin's responsibility ends at **"here are the changed paths, in Silo's terms."** Everything past that is the host engine, which is provider-agnostic.
|
||||
|
||||
## 5. The `scan_source.v1` contract (centerpiece)
|
||||
|
||||
A new, additive capability in `silo-plugin-sdk`. Existing plugins never declare it and are unaffected (§11).
|
||||
|
||||
### Protobuf (as implemented in silo-plugin-sdk)
|
||||
|
||||
```proto
|
||||
// scan_source.v1 — host pulls changed paths from a provider on a timer.
|
||||
service ScanSource {
|
||||
// Host calls this on each poll tick for a configured scan_source capability.
|
||||
rpc PollChanges(PollChangesRequest) returns (PollChangesResponse);
|
||||
}
|
||||
|
||||
message PollChangesRequest {
|
||||
string capability_id = 1; // which configured scan_source instance
|
||||
string marker = 2; // opaque; empty string on first run
|
||||
ResolvedConnection connection = 3; // resolved upstream credentials (see §8)
|
||||
}
|
||||
|
||||
// Concrete upstream credentials the host resolves and hands to the plugin each
|
||||
// poll, so the plugin stays credential-source-agnostic and stores no secrets.
|
||||
message ResolvedConnection {
|
||||
string base_url = 1;
|
||||
string api_key = 2;
|
||||
}
|
||||
|
||||
message PollChangesResponse {
|
||||
// Absolute changed paths in the provider/source namespace (RAW — NOT yet
|
||||
// rewritten). Files or directories. The host applies the source's rewrite
|
||||
// rules and validates before enqueueing.
|
||||
repeated string source_paths = 1;
|
||||
// Opaque continuation token. Host stores verbatim and echoes it back
|
||||
// on the next PollChanges. The plugin never assumes the host parses it.
|
||||
string next_marker = 2;
|
||||
}
|
||||
```
|
||||
|
||||
### Contract rules
|
||||
|
||||
- **Pull only.** The host owns the timer and calls `PollChanges`. The plugin never initiates.
|
||||
- **Opaque marker.** The host stores and echoes `next_marker` without interpreting it. arr uses an RFC3339 timestamp; a future provider may use a sequence number or event id — no host change required.
|
||||
- **First run.** `marker == ""` ⇒ the plugin starts from "now" (do not replay full history).
|
||||
- **Idempotent, at-least-once.** The host may re-issue the same `marker` if it failed to persist `next_marker`; providers must tolerate returning overlapping paths. The host's suppression layer (§7) absorbs duplicates.
|
||||
- **Error semantics.** A failed `PollChanges` (e.g. arr unreachable) ⇒ the host keeps the *old* marker and retries next tick; nothing is skipped or advanced.
|
||||
- **Raw source paths.** `source_paths` are in the provider's namespace, NOT rewritten. The host applies each source's path-rewrite rules (host-owned, per-source) before resolving to a library folder.
|
||||
- **Credentials per call.** The host resolves the source's connection (§8) and passes the concrete `{base_url, api_key}` in `PollChangesRequest.connection` on every poll. The plugin reads them from the request — it stores no credentials and never knows whether they came from its own config or a reused Requests link. Delivered over the local host↔plugin gRPC channel only.
|
||||
|
||||
The capability needs **only** `PollChanges` for v1. A `TestConnection`/validate RPC is explicitly deferred (YAGNI; connection validity surfaces as a failed poll).
|
||||
|
||||
## 6. Marker & delivery semantics
|
||||
|
||||
- One opaque marker stored per source, host-side.
|
||||
- Pull-only does **not** exclude push-style providers: an inotify/Ceph plugin maintains its own internal buffer of observed changes and flushes it (and advances its marker) on each `PollChanges`. Cost: up to one poll-interval of latency. This keeps a single uniform contract.
|
||||
|
||||
## 7. Host: AutoscanStore engine
|
||||
|
||||
Provider-agnostic. Per enabled source, per tick:
|
||||
|
||||
1. `paths, next := plugin.PollChanges(marker)`; on error, log + keep marker + continue.
|
||||
2. For each `dir` in `uniqueParentDirs(paths)`:
|
||||
- `target := scantrigger.Resolve(dir)`. Unresolvable (outside Silo's media folders) ⇒ quiet skip.
|
||||
- Suppression key `fmt.Sprintf("%d|%s", folderID, target.Path)` with debounce TTL; already-claimed ⇒ skip.
|
||||
- Accumulate `target`; record claimed key.
|
||||
3. `scanqueue.EnqueueScans(targets)`. On failure, release claims and **do not** advance the marker.
|
||||
4. On success, persist `next_marker`.
|
||||
|
||||
This is the salvaged core of PR #43 (`internal/autoscan/service.go` resolve/suppress/enqueue, `dedupe.go`, `suppress.go`), generalized so its input is "paths from a provider" rather than "paths from arr." Path-rewrite helpers (`rewrite.go`, `suggest.go`, `history.go`, the arr history client) leave the host and move into the plugin (§10).
|
||||
|
||||
## 8. Host: Connections (reuse or own)
|
||||
|
||||
A **connection** is "an arr server Silo can reach," created two ways:
|
||||
|
||||
- **Own:** name + base URL + API key, entered in the Autoscan category; the key is Fernet-encrypted at rest (same mechanism as existing service credentials — never plaintext in JSONB).
|
||||
- **Linked (reuse):** a live reference to an existing Requests `request_integrations` arr server. Resolved at use-time, so edits in Requests propagate.
|
||||
|
||||
**Resolution:** on each poll the host resolves the chosen connection (own or linked) into concrete `{base_url, api_key}` and passes them in `PollChangesRequest.connection` (§5). The plugin is credential-source-agnostic — it only ever sees resolved values, fresh each poll, so a change to a reused Requests credential propagates immediately with no re-push. This "reuse or own" behaviour is therefore free for every future provider. (Resolving per-call rather than caching in plugin config also means no decrypted secret is ever stored at rest in the plugin.)
|
||||
|
||||
**Decoupling from Requests:** the link is **soft and optional** — `SET NULL` / surfaced-as-"needs attention", never cascading. Requests has no knowledge of Autoscan. Autoscan and Requests are peers; Autoscan merely *offers* to borrow a Requests connection.
|
||||
|
||||
## 9. Host: Autoscan category & data model
|
||||
|
||||
New top-level **Autoscan** admin section (its own nav entry, not a Requests tab). Manages **Connections** (§8) and **Sources**.
|
||||
|
||||
A **source** corresponds to one configured `scan_source.v1` capability instance and carries:
|
||||
|
||||
- **Host-owned per-source config:** path rewrites (`autoscan_sources.path_rewrites`, edited in the Autoscan Sources UI) and any provider-specific settings. _(Revised post-merge: rewrites moved host-side; the plugin has no config.)_
|
||||
- **Host-owned per-source state:** the chosen connection, enabled on/off, poll interval (per-source, not one global timer), the opaque marker, and last-run status/error.
|
||||
|
||||
**Install flow + source auto-discovery (resolves §14 risk #1).** A scan-source plugin (Sonarr/Radarr, etc.) is installed through the **normal `/admin/plugins`** page — there is no separate install path; the installer doesn't special-case the capability type. Source rows are then never created by hand: the host enumerates every installed `scan_source.v1` capability (via the plugin installation store) and *seeds a disabled, connection-less row* per capability (`INSERT … ON CONFLICT (installation_id, capability_id) DO NOTHING`, never disturbing existing config). Discovery runs **both on each poll cycle AND when the Autoscan sources list is viewed** (so a freshly-installed plugin appears in Autoscan immediately, even when autoscan is disabled and nothing is polling). The operator then binds a connection and enables it. Consequently **`connection_id` is nullable**: a freshly-discovered source has none, and the engine skips any enabled source with no connection (recording a "no connection bound" status). The admin API rejects enabling a source that has no effective connection.
|
||||
|
||||
**Per-source interval** is honoured as a floor: the global poll task runs at `default_poll_interval_seconds`, and within a cycle the engine skips a source whose `last_run_at` is newer than its own `poll_interval_seconds` (falling back to the default). A source therefore polls *at most* every N seconds, never more often than the global cadence.
|
||||
|
||||
### Schema changes (host)
|
||||
|
||||
Rework the PR #43 tables; **no FK into `request_integrations`**:
|
||||
|
||||
- `autoscan_connections` — `id`, `name`, `kind` (e.g. `sonarr`/`radarr`), and *either* own credentials (`base_url`, `api_key_ref` encrypted) *or* a nullable soft link `request_integration_id` (`ON DELETE SET NULL`).
|
||||
- `autoscan_sources` — `id`, `(installation_id, capability_id)` (unique; the discovered identity), **nullable** `connection_id → autoscan_connections` (`ON DELETE RESTRICT`; null until the operator binds one), `enabled`, `poll_interval_seconds` (nullable; null = use default), `marker` (opaque text, nullable), `last_run_at`, `last_error`. Creation-time validity for connections (own creds OR a link) is enforced at the application layer (the DB CHECK would conflict with `ON DELETE SET NULL` on the reuse link).
|
||||
- `autoscan_settings` (optional) — retain a global enable + default interval; per-source interval overrides it.
|
||||
|
||||
A migration supersedes PR #43's `171_autoscan` schema (which is not yet upstream, so this is a forward redefinition, not a production migration of live data on `origin/main`).
|
||||
|
||||
## 10. Plugin: Sonarr/Radarr scan-source
|
||||
|
||||
A new installable repo structured like `silo-plugin-tmdb` (standalone Go module, `manifest.json` declaring `scan_source.v1`, gRPC `main.go`, depends on the new tagged `silo-plugin-sdk`). Receives resolved arr `{base_url, api_key}` as config. Implements `PollChanges`:
|
||||
|
||||
1. Treat `marker` as the "since" timestamp (empty ⇒ now).
|
||||
2. Call arr `/api/v3/history/since`, capped by a bounded window (24h max-lookback floor + overlap buffer — relocated from PR #43) so a long outage cannot pull an oversized response.
|
||||
3. Extract paths: **imports** (`downloadFolderImported.importedPath`) and **renames** (`episodeFileRenamed`/`movieFileRenamed`: both new `path` and old `sourcePath`). Deletes ignored (upgrade-deletes are covered by the paired import; standalone deletes carry no path).
|
||||
4. Return `{source_paths = the RAW arr paths, next_marker = newest history timestamp}`. The plugin does NOT rewrite — the host normalizes separators and applies the source's rewrite rules. _(Revised post-merge.)_
|
||||
|
||||
**Rewrite config + "Sync from arr":** the rewrite list and the suffix-match suggester (which matches arr root folders to Silo media folders) live in the plugin. The suggester needs Silo's media-folder list, obtained via the existing host library-listing service (`internal/pluginhost` library lister) exposed to plugins. The relocated logic is PR #43's `suggest.go`/`suggest_deps.go`/`rewrite.go`/`history.go`.
|
||||
|
||||
## 11. Backward compatibility (hard requirement)
|
||||
|
||||
Adding `scan_source.v1` **must not break existing plugins.** It doesn't, by construction:
|
||||
|
||||
- Capabilities are independent opt-in declarations. Each host subsystem **filters for its own type and ignores the rest** (verified: `metadata_provider.v1`, `scheduled_task.v1`, `event_consumer.v1`, … each use `if type != mine { skip }`). There is **no global allowlist that rejects unknown capability types.**
|
||||
- **Additive-only rule:** the SDK change *adds* new proto messages/service + a new capability string. It never modifies the shape of an existing capability. (Mutating an existing capability is the only thing that would break plugins; we forbid it.)
|
||||
- **No forced upgrades:** existing plugin repos stay pinned to their current SDK version and build unchanged. Only a plugin that *wants* to be a scan source adopts the new SDK version.
|
||||
|
||||
## 12. Build order & repo decomposition
|
||||
|
||||
One shared contract, three repos (+ catalog), in dependency order:
|
||||
|
||||
1. **`silo-plugin-sdk`** (long pole): define `scan_source.v1` (proto + generated code + manifest helper + runtime server scaffold), tag a new version (e.g. `v0.5.0`). The tagged release is a hard serialization point — host and plugin cannot build against the contract until it exists.
|
||||
2. **silo-server (host)** — bump SDK dep; build AutoscanStore (generalized), the `scan_source.v1` caller (timer → `PollChanges`), connections + resolution, the Autoscan category UI, retire the in-process arr autoscan, add the reworked migration. *Parallelizable with (3) once (1) is tagged.*
|
||||
3. **arr plugin (new repo)** — depends on SDK `v0.5.0`; implement `PollChanges`, rewrites, "Sync from arr", bounded window.
|
||||
4. **`silo-plugins`** (catalog) — add the entry so the arr plugin is installable.
|
||||
|
||||
Each repo gets its **own implementation plan** (separate `writing-plans` pass); this single spec defines the shared contract and per-side responsibilities they all build against.
|
||||
|
||||
**Local logistics:** only `silo-server` and `silo-plugin-tmdb` are currently checked out. Implementation will require `silo-plugin-sdk` and a new arr-plugin repo (and later `silo-plugins`) available locally.
|
||||
|
||||
## 13. Salvage map (from closed PR #43)
|
||||
|
||||
| PR #43 code | Destination |
|
||||
|---|---|
|
||||
| arr `/history` client, imports + renames, bounded window (`history.go`) | arr plugin |
|
||||
| path rewrite apply (`rewrite.go`) + "Sync from arr" suggester (`suggest.go`, `suggest_deps.go`) | arr plugin |
|
||||
| `last_poll_at` cursor | host, generalized to opaque per-source `marker` |
|
||||
| resolve + dedupe + suppress + enqueue (`service.go`, `dedupe.go`, `suppress.go`) | host AutoscanStore (generic) |
|
||||
| `171_autoscan` schema (FK → `request_integrations`) | host, reworked: `autoscan_connections` + `autoscan_sources`, no requests FK |
|
||||
| Autoscan admin tab under Requests (`AdminRequests.tsx`) | host, as its own Autoscan category |
|
||||
|
||||
## 14. Risks & open questions
|
||||
|
||||
1. **Plugin multiplicity for multiple arr servers.** Does the runtime support multiple configured instances of one plugin (one per arr server), or is config per-installation only? If the latter, "two Sonarrs + one Radarr" implies either multiple installations or a plugin that manages several connections internally. *Resolve before the host/plugin plans by reading `internal/plugins` installation + capability-instance handling.*
|
||||
2. **Plugin config UI richness.** The rewrite table + a "Sync from arr" action are richer than typical key-value plugin config. Verify the plugin config-schema/admin-config UI can host a structured list and an action button; if not, design a minimal affordance (e.g. suggester runs on first configure, or a dedicated plugin HTTP route via `http_routes.v1`).
|
||||
3. **Library-folder list exposure to plugins.** Confirm the `internal/pluginhost` library lister is reachable by a `scan_source` plugin for the suggester. If not, add a host service method.
|
||||
4. **Marker persistence vs at-least-once.** Ensure marker write happens only after successful enqueue, matching PR #43's "don't advance on failure" guarantee.
|
||||
5. **Closed PR disposition.** The in-process autoscan on local `main` (34 commits) should be removed/replaced rather than shipped; sequence the host plan to retire it cleanly.
|
||||
|
||||
## 15. Testing strategy
|
||||
|
||||
- **SDK:** proto compiles; manifest helper round-trips `scan_source.v1`; runtime scaffold serves `PollChanges`.
|
||||
- **Host engine:** table-driven tests for resolve→suppress→enqueue→marker (port PR #43's `service_test.go`, generalized to a fake `ScanSource` client): dedupe, distinct-paths-same-folder, disabled no-op, poll failure keeps marker, enqueue failure releases claims + keeps marker, opaque-marker pass-through. Run in the libvips-equipped environment (host handler tests link libvips via CGO — see workspace note).
|
||||
- **Connections:** resolution of own vs linked; linked falls back gracefully when the Requests row is deleted (`SET NULL`).
|
||||
- **arr plugin:** history parsing (imports + renames, ignores unrelated events), rewrite application, bounded window, first-run "from now"; suffix-match suggester cases (unique/ambiguous/covered/no-op/normalization) ported from PR #43.
|
||||
- **Compat:** an existing-style plugin manifest with no `scan_source.v1` is unaffected by host scan-source dispatch.
|
||||
|
||||
## 16. Out of scope / future
|
||||
|
||||
- inotify and Ceph-notify scan-source plugins (the contract accommodates them via the buffer-and-flush pattern of §6).
|
||||
- A push/event delivery mode, if pull latency ever proves insufficient.
|
||||
- A `TestConnection` RPC on the capability.
|
||||
- Migrating Requests onto a shared connection abstraction (today's reuse is a one-directional soft link only).
|
||||
@@ -0,0 +1,173 @@
|
||||
# Autoscan: Sync Path Rewrites from Arr Root Folders
|
||||
|
||||
**Status:** Design approved, ready for implementation plan
|
||||
**Date:** 2026-06-02
|
||||
**Area:** `internal/autoscan`, `internal/api`, `web/src/pages/AdminRequests.tsx`
|
||||
**Extends:** the autoscan feature (`docs/superpowers/specs/2026-06-02-autoscan-arr-polling-design.md`)
|
||||
|
||||
## Summary
|
||||
|
||||
Autoscan resolves an imported file path against Silo's media folders, but
|
||||
Radarr/Sonarr often report files under paths that differ from how Silo mounts
|
||||
the same content — so operators must hand-author per-root `path_rewrites`. For a
|
||||
multi-instance, many-root deployment this can be dozens of rows.
|
||||
|
||||
This adds a **"Sync rewrites from arr"** convenience: for one instance, query its
|
||||
root folders, match each root to a Silo media folder by **shared trailing path
|
||||
segments**, and present the resulting `{from, to}` rewrites as a **preview** the
|
||||
admin reviews and commits. Manual rewrites remain the source of truth; sync only
|
||||
*fills the editor* — nothing persists until the admin saves.
|
||||
|
||||
The matcher is purely structural (no assumptions about any particular path
|
||||
layout) so it helps any deployment whose arr and Silo share folder names at some
|
||||
depth, and degrades gracefully (no shared names → "unmatched" → manual entry)
|
||||
when they do not.
|
||||
|
||||
## Goals
|
||||
|
||||
- One-click suggestion of `path_rewrites` for an instance from its arr root folders.
|
||||
- Correct disambiguation when multiple Silo folders share a leaf name.
|
||||
- Preview-then-commit: the admin sees proposed/unmatched/ambiguous and saves
|
||||
through the existing per-source save. Manual rows are never removed or overwritten.
|
||||
- Deployment-agnostic: no hard-coded paths; works for any structure; surfaces
|
||||
match confidence so weak matches can be rejected.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No automatic/background writing of rewrites — the admin always commits.
|
||||
- No new persistence path — apply is the existing `PUT /sources/{id}`.
|
||||
- No case-insensitive matching (case-sensitive segments; correct on Linux). A
|
||||
toggle can be added later if needed.
|
||||
- No "sync all instances at once" — per-instance, matching the editor UI.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Matching function (core, pure)
|
||||
|
||||
A single side-effect-free function in `internal/autoscan`:
|
||||
|
||||
```go
|
||||
func suggestRewrites(arrRoots []string, siloFolderPaths []string, existing []PathRewrite) RewriteSuggestions
|
||||
```
|
||||
|
||||
```go
|
||||
type RewriteSuggestions struct {
|
||||
Proposed []ProposedRewrite `json:"proposed"`
|
||||
Unmatched []string `json:"unmatched"`
|
||||
Ambiguous []AmbiguousRoot `json:"ambiguous"`
|
||||
Covered []string `json:"covered"`
|
||||
}
|
||||
type ProposedRewrite struct {
|
||||
From string `json:"from"` // arr root
|
||||
To string `json:"to"` // matched Silo folder path
|
||||
MatchDepth int `json:"match_depth"` // trailing segments shared (confidence)
|
||||
}
|
||||
type AmbiguousRoot struct {
|
||||
Root string `json:"root"`
|
||||
Candidates []string `json:"candidates"`
|
||||
}
|
||||
```
|
||||
|
||||
**Normalization** (applied to every path before comparison): backslashes → `/`
|
||||
(reuse `normalizeSeparators`), strip a trailing `/`, collapse duplicate slashes.
|
||||
This makes any path format (POSIX, Windows, single- or many-root) comparable.
|
||||
|
||||
**Per arr root:**
|
||||
1. **Covered-first:** if any `existing` rewrite's `From` already matches the root
|
||||
(exact, or root starts with `From + "/"`, using the same boundary rule as
|
||||
`applyRewrites`), classify as **Covered** and skip — so sync never re-proposes
|
||||
what a manual or broad rule already handles.
|
||||
2. Split both the root and each Silo folder path into segments; compute the
|
||||
**longest common trailing-segment suffix** (full-segment equality, so `4ktv7`
|
||||
does not match `4ktv70`).
|
||||
3. Among Silo folders, take the maximum suffix length `> 0` and the folders that
|
||||
achieve it:
|
||||
- exactly one → **Proposed** `{From: root, To: folder, MatchDepth: n}`
|
||||
- more than one → **Ambiguous** `{root, candidates}`
|
||||
- zero → **Unmatched**
|
||||
|
||||
No deployment constants appear in the function; it operates only on the string
|
||||
slices it is given.
|
||||
|
||||
### 2. Preview endpoint & service
|
||||
|
||||
**Endpoint:** `GET /api/v1/admin/autoscan/sources/{id}/rewrite-suggestions` →
|
||||
`RewriteSuggestions` JSON (admin-gated, like the other autoscan routes).
|
||||
|
||||
**Service method** `Service.SuggestRewrites(ctx, integrationID) (RewriteSuggestions, error)`:
|
||||
1. `Repository.GetSource(id)` — a new single-source variant of the existing join
|
||||
that returns `kind`/`base_url`/`api_key_ref`/`path_rewrites` even when the
|
||||
source is disabled. Not found → `ErrIntegrationNotFound`.
|
||||
2. Decrypt the API key via the existing `SecretResolver`.
|
||||
3. `RootFolderClient.RootFolders(ctx, baseURL, apiKey)` → arr root paths.
|
||||
4. `FolderLister.ListFolderPaths(ctx)` → all Silo media-folder paths.
|
||||
5. Return `suggestRewrites(arrRoots, siloPaths, source.PathRewrites)`.
|
||||
|
||||
**Two new narrow `Service` dependencies** (injected like `history`/`resolver`):
|
||||
- `RootFolderClient.RootFolders(ctx, baseURL, apiKey string) ([]string, error)` —
|
||||
an `arrclient`-backed impl reusing `arrclient.ListRootFolders` (returns paths only).
|
||||
- `FolderLister.ListFolderPaths(ctx) ([]string, error)` — a thin adapter over
|
||||
`catalog.FolderRepository.List()` flattening each folder's `Paths`.
|
||||
|
||||
**Wiring:** both constructed where the autoscan handler is built in `router.go`
|
||||
(`deps.FolderRepo` already in scope; the arr client is trivial). The handler's
|
||||
service interface gains `SuggestRewrites`.
|
||||
|
||||
**Errors:** instance not found → 404; arr unreachable / bad key → 502 with the
|
||||
arr error surfaced; Silo folder-list failure → 500.
|
||||
|
||||
No new write path — the frontend merges `proposed` into the editor and the admin
|
||||
saves via the existing `PUT /sources/{id}`.
|
||||
|
||||
### 3. Frontend (preview UI)
|
||||
|
||||
In `AutoscanSourceEditor` (`web/src/pages/AdminRequests.tsx`), add a **"Sync from
|
||||
arr"** button beside Save. On click, a new hook
|
||||
(`useAutoscanRewriteSuggestions`) calls the endpoint and opens a preview panel:
|
||||
|
||||
- **Proposed** — each row: `From → To` + a **confidence badge** from `match_depth`
|
||||
(e.g. "3 segments" strong, "1 segment — weak"), with a checkbox (default checked).
|
||||
- **Unmatched** — info-only list of arr roots with no Silo match ("add manually").
|
||||
- **Ambiguous** — arr root + candidate Silo paths (info-only; admin adds a manual row).
|
||||
|
||||
**"Add selected to rewrites"** merges the checked proposals into the editor's
|
||||
existing rewrite rows, **deduped by `from`, never removing manual rows**. The
|
||||
admin then clicks the normal **Save**. New types in `web/src/api/types.ts`
|
||||
(`AutoscanRewriteSuggestions`, `ProposedRewrite`, `AmbiguousRoot`) and hook in
|
||||
`web/src/hooks/queries/useAutoscan.ts`. Reuse existing
|
||||
`Button`/`Badge`/`Dialog`/`Field` primitives.
|
||||
|
||||
### 4. Testing
|
||||
|
||||
- **`suggestRewrites`** — table-driven, fixtures spanning layouts: multi-segment
|
||||
match, single-segment unique, longest-suffix disambiguation (`Anime/Subs` vs
|
||||
`Anime2/Subs`), unmatched (no shared segment), already-covered via a broad
|
||||
prefix rule, an ambiguous tie, plus **unlike-this-deployment** cases
|
||||
(`/data/Movies` arr vs `/library/Films` Silo → unmatched; `/srv/tv/Show` vs
|
||||
`/tank/television/Show` → leaf match) and normalization (trailing slash,
|
||||
backslashes, duplicate slashes).
|
||||
- **`Service.SuggestRewrites`** — fakes for source store / `RootFolderClient` /
|
||||
`FolderLister` / secrets: asserts proposed/unmatched/ambiguous; arr-unreachable
|
||||
surfaces the error.
|
||||
- **Handler** — returns the JSON; 404 for missing instance; 502 when the arr is
|
||||
unreachable.
|
||||
- **Frontend** — the button hits the endpoint; the preview renders all three
|
||||
buckets with confidence badges; "Add selected" merges deduped rows; Save
|
||||
persists via the existing mutation.
|
||||
|
||||
## Risks & open considerations
|
||||
|
||||
- **Weak (1-segment) matches** can be coincidental across very different layouts;
|
||||
mitigated by surfacing `match_depth` so the admin rejects them before saving.
|
||||
- **Case-sensitivity** means case-mismatched mounts produce Unmatched rather than
|
||||
a wrong auto-match — a deliberate safe default; revisit only if requested.
|
||||
- **Arr reachability** is required for sync (unlike the rest of autoscan which
|
||||
polls on a schedule); a 502 with the arr error tells the admin to check the
|
||||
instance, and manual entry still works.
|
||||
|
||||
## References
|
||||
|
||||
- Reused: `internal/requests/arrclient.ListRootFolders`,
|
||||
`internal/catalog/FolderRepository.List`, `internal/autoscan` (`PathRewrite`,
|
||||
`normalizeSeparators`, `Repository`, `Service`, the existing
|
||||
`PUT /autoscan/sources/{id}` save).
|
||||
@@ -0,0 +1,161 @@
|
||||
# Autoscan source labels — design
|
||||
|
||||
**Date:** 2026-06-05
|
||||
**Status:** Approved, pending implementation plan
|
||||
**Scope:** Admin UI labeling of autoscan scan sources, generic across plugins, plus an operator-editable per-source label.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
Autoscan sources are labeled in the admin UI (Sources tab and Activity tab) by their
|
||||
`capability_id` plus `plugin #<installation_id>`. The `arr` scan-source plugin fans out
|
||||
one source per connection (Sonarr, Sonarr4k, Radarr, Radarr4k) under a single generic
|
||||
`arr` capability, so every row rendered an identical "arr (plugin #4)" label. A prior
|
||||
change made the Sources/Activity labels lead with the bound connection name, which fixes
|
||||
`arr`, but the fallback for any other plugin is still the cryptic `capability_id` +
|
||||
`plugin #N`.
|
||||
|
||||
Two gaps remain:
|
||||
|
||||
1. **Other plugins look cryptic.** A connectionless plugin (e.g. cephfs) or any future
|
||||
plugin still falls back to the raw `capability_id`, ignoring the human-friendly
|
||||
`display_name` the plugin already declares in its manifest.
|
||||
2. **No operator control.** An operator cannot name a source themselves. This is the only
|
||||
fully-general answer for plugin shapes the automatic chain cannot disambiguate (e.g. a
|
||||
single plugin installation creating several connectionless sources distinguished only
|
||||
by their `source_config`).
|
||||
|
||||
## Goals
|
||||
|
||||
- Generic, plugin-agnostic labels driven by metadata that already exists
|
||||
(manifest `display_name`, connection `name`), so future plugins look good with **no
|
||||
host code per plugin**.
|
||||
- An optional operator-editable label per source that overrides the automatic chain.
|
||||
- A single shared label-resolution helper, consumed by both the Sources and Activity
|
||||
panels, so the logic is not duplicated.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Plugin-declared label templates in the manifest/SDK (the "option C" from brainstorming).
|
||||
Deferred: it requires a cross-repo SDK/manifest contract change. The chain below leaves a
|
||||
clean seam for it.
|
||||
- Backend-side label resolution returned on API payloads. The resolution stays in the
|
||||
frontend helper for now; promote it to the backend only if the mobile clients need the
|
||||
same labels.
|
||||
|
||||
## Resolution chain
|
||||
|
||||
A source's display label resolves through four rungs, most-specific first:
|
||||
|
||||
```
|
||||
operator label -> connection name -> manifest display_name -> capability_id
|
||||
```
|
||||
|
||||
- **operator label** — the optional per-source text the admin sets (this design's new field).
|
||||
- **connection name** — the operator-named `autoscan_connections.name` bound to the source.
|
||||
- **manifest display_name** — from the plugin manifest, surfaced by the
|
||||
`scan-source-plugins` endpoint (`AutoscanAvailableSource.display_name`).
|
||||
- **capability_id** — last-resort raw identifier; never worse than today.
|
||||
|
||||
The chosen rung is the **title**. The remaining identity (plugin display name / capability +
|
||||
`plugin #<installation_id>`) renders as a **subtitle**, so no information is lost when a
|
||||
higher rung wins. Example: operator label `4K Movies` as title, `Radarr4k · plugin #4` as
|
||||
subtitle.
|
||||
|
||||
### Per-plugin behavior
|
||||
|
||||
| Source shape | Title | Subtitle |
|
||||
| --- | --- | --- |
|
||||
| arr, connection bound, no operator label | `Radarr4k` (connection) | `arr · plugin #4` |
|
||||
| arr, operator label set | `4K Movies` (operator) | `Radarr4k · plugin #4` |
|
||||
| cephfs, no connection | `CephFS Watcher` (manifest display_name) | `plugin #5` |
|
||||
| future plugin, nothing declared | `capability_id` | `plugin #N` |
|
||||
| Activity event whose source was deleted (`source_id` null) | manifest display_name or `capability_id` from the event's stored fields | `plugin #N` |
|
||||
|
||||
## Components
|
||||
|
||||
### Shared helper — `web/src/lib/autoscanLabels.ts` (new)
|
||||
|
||||
Pure, React-free, unit-tested.
|
||||
|
||||
- `composeSourceLabel({ operatorLabel, connectionName, displayName, capabilityId, installationId })`
|
||||
`=> { name: string; detail: string }` — implements the four-rung chain and the
|
||||
subtitle composition.
|
||||
- `pluginDisplayNameKey(installationId, capabilityId) => string` — stable map key.
|
||||
- `buildPluginDisplayNames(available: AutoscanAvailableSource[]) => Map<string, string>` —
|
||||
builds the `installation_id + capability_id -> display_name` lookup both panels use.
|
||||
|
||||
Future seam: `composeSourceLabel` may later accept an optional `configLabel` rung (for the
|
||||
deferred plugin-template option) without changing either panel's call shape.
|
||||
|
||||
### Migration — `migrations/174_autoscan_source_label.{up,down}.sql` (new)
|
||||
|
||||
```sql
|
||||
-- up
|
||||
ALTER TABLE public.autoscan_sources ADD COLUMN label text NOT NULL DEFAULT '';
|
||||
-- down
|
||||
ALTER TABLE public.autoscan_sources DROP COLUMN label;
|
||||
```
|
||||
|
||||
`label` empty string = unset; the resolution chain falls through to the automatic rungs.
|
||||
174 is a fresh, unused migration number (no collision with the reused-172 history).
|
||||
|
||||
### Backend (Go)
|
||||
|
||||
- `internal/autoscan/types.go` — add `Label string` to the `Source` struct and the source
|
||||
input type.
|
||||
- `internal/autoscan/repository.go` — add `label` to `sourceColumns`, `scanSource`, and the
|
||||
create/update statements.
|
||||
- `internal/api/handlers/autoscan.go` — accept `label` on source update; include it in the
|
||||
`autoscanStatusSource` and source responses. Trim whitespace and cap length at 120
|
||||
characters server-side.
|
||||
|
||||
### Frontend
|
||||
|
||||
- `web/src/api/types.ts` — add `label` to `AutoscanSource` and `AutoscanSourceInput`.
|
||||
- `web/src/pages/admin/autoscan/SourcesPanel.tsx` — replace the inline `sourceIdentity`
|
||||
block with a call to `composeSourceLabel`. Add an optional "Label" text input to the
|
||||
source row, saved on blur via the existing `fullBody` PATCH path (mirrors the interval
|
||||
field's `handleIntervalBlur` auto-save). Build the display-name map from the existing
|
||||
`useAvailableScanSources()` query.
|
||||
- `web/src/pages/admin/autoscan/ActivityPanel.tsx` — add `useAvailableScanSources()`, build
|
||||
the display-name map, map `source_id -> source` to obtain the operator label and
|
||||
connection, and route `pollSourceName` / `scanSourceName` through `composeSourceLabel`,
|
||||
returning `.name`. Preserve the existing `"Autoscan"` fallback for scans with no
|
||||
capability.
|
||||
|
||||
## Edit UX
|
||||
|
||||
Inline per-row, reusing the existing on-blur auto-save pattern (no modal, no new
|
||||
interaction model). The label input is blank by default with placeholder text indicating
|
||||
it is optional. Saving sends the full row state through the same source-update mutation
|
||||
used for connection/interval/rewrites.
|
||||
|
||||
## Error handling and edge cases
|
||||
|
||||
- **Deleted source on an Activity event** — `autoscan_events.source_id` is
|
||||
`ON DELETE SET NULL`; with a null `source_id` there is no operator label or connection,
|
||||
so the chain resolves from the event row's own stored `capability_id` / `installation_id`
|
||||
(plus manifest display_name if the plugin is still installed).
|
||||
- **Uninstalled plugin** — `display_name` does not resolve; the chain falls to
|
||||
`capability_id`.
|
||||
- **Bound connection** — `autoscan_sources.connection_id` is `ON DELETE RESTRICT`, so a
|
||||
bound connection always resolves to a name.
|
||||
- **Length / whitespace** — the operator label is trimmed and capped at 120 characters
|
||||
server-side; an all-whitespace label normalizes to empty (unset).
|
||||
|
||||
## Testing
|
||||
|
||||
- `web/src/lib/autoscanLabels.test.ts` — covers each rung of the chain: operator label
|
||||
wins, connection name wins, manifest display_name wins, capability fallback, and the
|
||||
subtitle composition.
|
||||
- Extend the existing autoscan repository/handler Go tests to cover the `label` column
|
||||
round-trip (create/update/read) and the server-side trim + length cap.
|
||||
|
||||
## Scope summary
|
||||
|
||||
One migration pair, three Go files, four frontend files (one new). No SDK changes, no
|
||||
multi-repo coordination. Operator-editable labels (this design) and the automatic chain
|
||||
share the same helper, so the deferred plugin-template option layers on later with no
|
||||
rework.
|
||||
@@ -0,0 +1,472 @@
|
||||
# Marker sources & TheIntroDB contribution — design
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Status:** Draft (rev 2), pending review
|
||||
**Scope:** Review the current TheIntroDB marker integration, formalize a multi-source
|
||||
marker-provider model (query-all / best-wins), and add the ability to contribute Silo's own
|
||||
markers back to TheIntroDB. This round designs the **server-side API and data model only** —
|
||||
web-admin and mobile surfaces are deferred to a UI-only follow-up. No client (Android/Apple)
|
||||
changes required.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
**Rev 2 changes:** contribution is configured **per provider** (off by default); all new web
|
||||
UI is **deferred** and the backend API is specified in full instead.
|
||||
|
||||
## Decisions locked for this design
|
||||
|
||||
These were chosen up front and constrain everything below:
|
||||
|
||||
1. **Deliverable now:** design/plan only. No source changes land from this document; it is
|
||||
the blueprint a follow-up implementation plan expands.
|
||||
2. **What we contribute:** *both* local auto-detected markers *and* manual corrections. Local
|
||||
detection only produces **intro markers for episodes**, so credits/recap/preview and all
|
||||
movies can only be contributed via manual correction — this asymmetry drives the design.
|
||||
3. **Account model:** a single server-wide TheIntroDB account, reusing the existing
|
||||
`introdb.api_key` server setting. Per-user accounts are a noted future seam, not built now.
|
||||
4. **Multi-source dispatch:** query all enabled online sources and keep the highest-quality
|
||||
result per segment ("best wins"). This makes *real per-segment confidence* (gap #2 below)
|
||||
a hard prerequisite, not an optional nicety.
|
||||
5. **Contribution is configured per provider, off by default.** Every contribution-capable
|
||||
provider has its own enable switch and submit settings, so when more providers exist an
|
||||
operator chooses *which* ones to submit to. Both the master contribute switch and the
|
||||
auto-submit-local-detections switch default to off; a single setting flips each on.
|
||||
6. **API-first, UI deferred.** This round fully specifies the backend (manual marker
|
||||
read/write/clear, contribution trigger + status, per-provider provider config, key
|
||||
validation and stats) so any future surface — web admin or mobile — builds on a complete
|
||||
contract. No new web UI is designed here beyond noting where it will eventually attach.
|
||||
|
||||
## Problem
|
||||
|
||||
Silo currently detects intro/recap/credits/preview markers from two places — a local
|
||||
chromaprint/chapter analyzer (`internal/intromarkers`) and the online TheIntroDB provider
|
||||
(`internal/markers/introdb`) — reconciled by a source-priority ladder at write time. The
|
||||
online side is cleanly abstracted behind a `markers.Provider` interface and a
|
||||
`markers.Registry`, which is good, but three things are missing:
|
||||
|
||||
1. The current TheIntroDB provider has **correctness gaps** measured against the real v3 API
|
||||
contract (TVDB lookups dropped, real confidence discarded — see "Review findings").
|
||||
2. There is **no path to contribute** Silo's markers back to TheIntroDB, even though the v3
|
||||
API exposes a submission endpoint and the package was deliberately built read-only.
|
||||
3. The provider abstraction is **fetch-only and first-hit**, so it does not yet support
|
||||
"several online sources, keep the best answer" or "this source can also accept
|
||||
contributions, and I choose which sources to submit to."
|
||||
|
||||
## Review findings — current implementation
|
||||
|
||||
### What is already right (keep it)
|
||||
|
||||
- **Clean source abstraction.** `markers.Provider` (`ID()`, `FetchMarkers()`) + `markers.Registry`
|
||||
with `FetchFirstHit` (`internal/markers/types.go`). TheIntroDB is *not* hardcoded; a second
|
||||
provider is one `Register()` call (`cmd/silo/main.go`).
|
||||
- **Multi-source-aware schema.** `media_files` carries per-segment provenance for all four
|
||||
kinds: `{intro,credits,recap,preview}_markers_{source,provider,confidence,algorithm,detected_at}`
|
||||
(migrations 108/109/137).
|
||||
- **Write-time reconciliation across source classes.** `CanWriteMarker`
|
||||
(`internal/markers/write.go`) + `MarkerSourcePriority` (`internal/models/marker_source.go`):
|
||||
`manual(4) > online/plugin(3) > s3(2) > scanner(1)`, confidence breaks ties. Local detection
|
||||
uses its own priority-gated path (`intromarkers.Repository.PatchIntroMarker`).
|
||||
- **Solid HTTP hygiene** in `internal/markers/introdb/client.go`: conservative rate limiter
|
||||
(2 req/s, burst 5; documented limit is 30 req/10s), retry with exponential backoff,
|
||||
`Retry-After` handling, 24h TTL cache with negative caching, hot-reloadable API key over
|
||||
Redis pub/sub.
|
||||
|
||||
### Gaps (measured against TheIntroDB's own v3 client + Jellyfin plugin)
|
||||
|
||||
| # | Gap | Evidence | Impact |
|
||||
|---|-----|----------|--------|
|
||||
| 1 | **TVDB lookups silently dropped.** Provider reads only TMDB+IMDB; client can't send `tvdb_id`. | `internal/markers/introdb/provider.go` (reads `ExternalIDKeyTMDB`/`IMDB` only); `client.go` `FetchEpisode/FetchMovie` build `tmdb_id` else `imdb_id`. | Titles matched only by TVDB (very common for anime / TheTVDB-first libraries) get **zero** markers, even though the resolver already populates `TvdbID` and the v3 API + both official clients support `tvdb_id`. Correctness bug. |
|
||||
| 2 | **Real confidence + submission_count discarded; hardcoded 0.9.** | `internal/markers/introdb/types.go` `segmentTimestamps` has only `start_ms`/`end_ms`; `pickMarker` sets `Confidence: 0.9`. | The `/media` response carries per-segment `confidence` and `submission_count`. Without them, "best wins" merging can't rank sources, and write-time confidence tie-breaks are meaningless. Hard blocker for the chosen dispatch model. |
|
||||
| 3 | **Naive multi-candidate selection.** `pickMarker` takes the *first* usable entry. | `internal/markers/introdb/provider.go`. | When several candidates remain (e.g. no duration match), first-usable may not be the most-submitted/highest-confidence. Minor today (duration narrows the set), but compounds once we rank by confidence. |
|
||||
| 4 | **No contribution path** (by design). | `internal/markers/introdb/types.go` package doc: "Submissions are intentionally not supported." | The whole point of this effort. |
|
||||
| 5 | **Provider is fetch-only.** No submit capability in the interface. | `markers.Provider` in `internal/markers/types.go`. | Contribution needs an optional `Submitter` capability so non-contributing sources don't have to implement it, and so submission is per provider. |
|
||||
| 6 | **No key validation / contribution stats.** | `IntroDBCredentialCard` in `web/src/pages/admin-settings/IntegrationsSettings.tsx` only sets the key. | v3 exposes `GET /v3/user/stats` to validate a key and show contribution totals/streaks. Useful "connect & verify" probe, currently absent from the API. |
|
||||
| 7 | **Single collapsed confidence in the write payload.** `BuildUpdatePayload` promotes one max confidence across all segments. | `internal/markers/write.go`. | The DB has *per-segment* confidence columns, but a merged result (intro from source A, credits from source B) can't populate them independently until per-segment confidence is threaded through the payload. Coupled to gap #2. |
|
||||
|
||||
## Authoritative API contract (TheIntroDB v3)
|
||||
|
||||
Confirmed from TheIntroDB's own `theintrodb` npm client (`src/funcs.ts`, `src/types.ts`) and
|
||||
the official Jellyfin plugin (`TheIntroDbClient.cs`). Base URL: `https://api.theintrodb.org/v3`.
|
||||
|
||||
### `GET /media` — read (public; optional key)
|
||||
|
||||
Query: one of `tmdb_id` / `tvdb_id` / `imdb_id` (preference order tmdb → tvdb → imdb),
|
||||
plus `season` + `episode` together for TV, plus optional `duration_ms` (selects the closest
|
||||
release version). Optional `Authorization: Bearer <key>` additionally includes that user's
|
||||
*pending* submissions.
|
||||
|
||||
Response — each segment kind is an array of:
|
||||
```jsonc
|
||||
{ "start_ms": 30000|null, "end_ms": 90000|null, "confidence": 0.93|null, "submission_count": 7|null }
|
||||
```
|
||||
`start_ms:null` = "from the beginning" (intro/recap); `end_ms:null` = "to the end"
|
||||
(credits/preview). Arrays may hold multiple entries (multiple release versions) — pick by
|
||||
duration match, then by `submission_count`/`confidence`.
|
||||
|
||||
### `POST /submit` — contribute (requires key)
|
||||
|
||||
`Authorization: Bearer <user key>` required; submissions are credited to that account.
|
||||
Body (one segment per call):
|
||||
```jsonc
|
||||
{
|
||||
"tmdb_id": 1234, // required
|
||||
"imdb_id": "tt0903747", // optional
|
||||
"type": "tv" | "movie",
|
||||
"segment": "intro" | "recap" | "credits" | "preview",
|
||||
"season": 1, // TV only (number, or "1,2,3" multi-select)
|
||||
"episode": 1, // TV only
|
||||
"video_duration_ms": 2760000, // optional, strongly recommended (release-version match)
|
||||
"start_ms": 0|null, // intro/recap may be null (= beginning)
|
||||
"end_ms": 90000|null // credits/preview may be null (= end)
|
||||
}
|
||||
```
|
||||
Response: `{ "submissions": [ { "id": uuid, "status": "pending"|"accepted"|"rejected",
|
||||
"weight": number, ... } ] }`. Submissions enter as `pending`; the backend weight-averages
|
||||
timestamps across submissions and moderates to accepted/rejected.
|
||||
|
||||
Validation rules to mirror client-side: intro/recap require an **end**; credits/preview
|
||||
require a **start**; TV requires season+episode; movies omit them; max timestamp 6h
|
||||
(21,600,000 ms); `video_duration_ms` is 0 (unknown) or ≥ 300,000 ms.
|
||||
|
||||
### `GET /user/stats` — validate key + contribution stats
|
||||
|
||||
`Authorization: Bearer <key>`. Returns `{ total, accepted, pending, rejected,
|
||||
acceptance_rate, current_streak, best_streak, total_time_saved_ms, top_media }`. A 2xx with no
|
||||
`error` means the key is valid. This is both our key-validation probe and the stats display.
|
||||
|
||||
### Rate / usage limits
|
||||
|
||||
429 responses carry `X-RateLimit-{Limit,Remaining,Reset}` and `X-UsageLimit-{Limit,Remaining,Reset}`.
|
||||
Reads count against the rate limit; **submissions count against the usage limit**. The
|
||||
contribution pipeline must honor `X-UsageLimit-Reset` and back off — reuse the existing
|
||||
limiter and add usage-limit awareness.
|
||||
|
||||
## Target architecture
|
||||
|
||||
Three layers, each independently shippable. Layer A is pure correctness and unblocks B and C.
|
||||
|
||||
### Layer A — Correctness foundation (gaps #1, #2, #3, #7)
|
||||
|
||||
- **TVDB in the client + provider.** Add `tvdb_id` to the query builder in
|
||||
`internal/markers/introdb/client.go` (`FetchEpisode`/`FetchMovie` gain a `tvdbID` arg,
|
||||
preference tmdb → tvdb → imdb, cache keys extended). Provider reads
|
||||
`req.ExternalIDs[markers.ExternalIDKeyTVDB]`. Resolver already supplies it — no resolver
|
||||
change.
|
||||
- **Capture confidence + submission_count.** Extend `segmentTimestamps` with
|
||||
`Confidence *float64` and `SubmissionCount *int`; add `SubmissionCount int` to
|
||||
`markers.Marker`; `pickMarker` uses the real confidence (fallback to a named default
|
||||
constant, e.g. `0.9`, only when the field is null) instead of hardcoding.
|
||||
- **Best-candidate selection.** `pickMarker` ranks candidates by `(submission_count desc,
|
||||
confidence desc)` after duration filtering, instead of first-usable.
|
||||
- **Per-segment confidence in the payload.** Replace the single `Confidence *float64` on
|
||||
`markers.MarkerUpdatePayload` with per-segment confidence (e.g. `IntroConfidence`,
|
||||
`CreditsConfidence`, …) so a merged multi-source result writes the correct
|
||||
`{kind}_markers_confidence` per kind. Update `BuildUpdatePayload` and the
|
||||
`scanner.FileRepository.UpsertMarkers` column mapping accordingly.
|
||||
|
||||
### Layer B — Multi-source dispatch ("query all, best wins")
|
||||
|
||||
- **`Submitter` capability (optional interface).**
|
||||
```go
|
||||
// internal/markers/types.go
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
// FetchUserStats validates the configured key and returns contribution stats.
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
Only `introdb.Provider` implements it; the contribution service type-asserts
|
||||
`provider.(markers.Submitter)`. This is also what makes contribution per-provider: the
|
||||
service only ever submits to registered providers that satisfy `Submitter` *and* are enabled
|
||||
in `marker_provider_config` (C6).
|
||||
|
||||
- **Merge dispatch.** Add `Registry.FetchMerged(ctx, req) (Result, bool, error)` that fans out
|
||||
to all *fetch-enabled* providers concurrently (bounded), then merges **per segment kind**
|
||||
keeping the candidate with the highest `(submission_count, confidence)`. `FetchFirstHit`
|
||||
stays as the trivial single-provider path / fallback. With only TheIntroDB enabled,
|
||||
`FetchMerged` behaves identically to today — zero behavior change until a second source
|
||||
exists.
|
||||
|
||||
- **Cross-source-class reconciliation is unchanged.** `FetchMerged` reconciles *within* the
|
||||
online class; `CanWriteMarker` still arbitrates online vs scanner vs manual at write time.
|
||||
No double logic.
|
||||
|
||||
- **Provider enable/order config.** Driven by the `marker_provider_config` table (C6):
|
||||
`fetch_enabled` selects which providers are queried, `fetch_priority` is the merge
|
||||
tiebreaker. The metadata `library_provider_chains` DB pattern (`internal/metadata/chain.go`)
|
||||
is the upgrade path *if* per-library control is ever needed — noted, not built.
|
||||
|
||||
- **Future plugin sources (deferred seam).** A `marker_provider.v1` plugin capability would let
|
||||
third-party sources plug in. The clean seam is a `PluginMarkerProvider` adapter implementing
|
||||
`markers.Provider` (mirroring `internal/metadata/plugin_provider.go`), so the registry never
|
||||
changes. Cross-repo (silo-plugin-sdk) — explicitly out of scope here.
|
||||
|
||||
### Layer C — Contribution to TheIntroDB
|
||||
|
||||
#### C1. Submission client
|
||||
|
||||
Add to `internal/markers/introdb/client.go`:
|
||||
- `SubmitSegment(ctx, SubmissionRequest) (SubmissionResult, error)` → `POST /v3/submit`,
|
||||
Bearer key **required** (error early if `introdb.api_key` is empty), usage-limit aware.
|
||||
- `FetchUserStats(ctx) (UserStats, error)` → `GET /v3/user/stats`.
|
||||
`introdb.Provider` implements `markers.Submitter` by delegating to these.
|
||||
|
||||
#### C2. What is eligible to contribute
|
||||
|
||||
A marker is contributable iff:
|
||||
- the target provider is a registered `Submitter` with `contribute_enabled = true` (C6), and
|
||||
- its segment source ∈ `{scanner, manual}` (**never** `online` — that came *from* an online
|
||||
provider; contributing it back is circular), and
|
||||
- the file resolves to a usable external ID (tmdb/tvdb/imdb) via the existing resolver, and
|
||||
- for **auto** contribution: provider `contribute_auto_local = true`, source = `scanner`,
|
||||
kind = `intro`, confidence ≥ that provider's `contribute_min_confidence`. (Local detection is
|
||||
intro-only/episodes-only.)
|
||||
- for **manual** contribution: any kind, episodes or movies, source = `manual`.
|
||||
|
||||
#### C3. Manual marker API (write path; UI deferred)
|
||||
|
||||
Markers live on `media_files`, and an item can have several versions, so the manual API is
|
||||
**file-scoped** for precision, with an item-level convenience resolver. All under the existing
|
||||
`RequireAdmin` group (`internal/api/middleware/auth.go`). New `internal/api/handlers/admin_markers.go`:
|
||||
|
||||
- `GET /admin/files/{fileId}/markers` — current per-segment values **plus full provenance**
|
||||
(`source`, `provider`, `confidence`, `algorithm`, `detected_at`) for each of
|
||||
intro/recap/credits/preview. The read side an editor (or a script) needs.
|
||||
- `PUT /admin/files/{fileId}/markers` — upsert the **manual** layer. Body carries, per segment,
|
||||
either `{start, end}` (seconds) or `null` to clear that segment's manual value. Writes
|
||||
`source = manual` (priority 4, beats every other source) through the existing priority-gated
|
||||
`scanner.FileRepository.UpsertMarkers` path, then notifies live sessions via
|
||||
`playback.MarkerUpdateNotifier.MarkersUpdated` (mirrors `admin_intro.go`).
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` — explicit single-segment clear.
|
||||
- `GET|PUT /admin/items/{id}/markers` — item-level convenience that resolves to the primary
|
||||
file (and reports the per-version set), for callers that don't track file IDs.
|
||||
|
||||
**Clear semantics.** Clearing a manual segment nulls that segment's value and provenance;
|
||||
`markers_source` drops to the next-highest segment still present. A subsequent local
|
||||
re-detection or online lazy-fetch can repopulate it. (We can't "un-overwrite" a value a manual
|
||||
edit replaced, so clearing means "remove the manual value," not "restore the previous source.")
|
||||
|
||||
**Validation** mirrors the contribution rules so a manual edit can always be contributed:
|
||||
`end > start`; intro/recap may omit start (= 0); credits/preview may omit end (= file
|
||||
duration); values within [0, file duration].
|
||||
|
||||
A web editor (player-chrome control with a "use current time" helper) attaches to this API
|
||||
later; its UX is intentionally **out of scope** for this round.
|
||||
|
||||
#### C4. Contribution tracking (idempotency + audit)
|
||||
|
||||
The audit log that makes submission idempotent and lets us report status. Per-provider by
|
||||
construction (the `provider` column), so the same marker can be tracked/submitted to several
|
||||
providers independently.
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_contributions.up.sql (next free number at implementation time)
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL, -- 'introdb'
|
||||
segment_kind text NOT NULL, -- intro|recap|credits|preview
|
||||
source text NOT NULL, -- what we contributed: 'scanner'|'manual'
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL, -- hash(segment_kind,start_ms,end_ms,duration_ms)
|
||||
submission_id uuid, -- returned by /submit
|
||||
status text NOT NULL, -- pending|accepted|rejected|error
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
`content_hash` over the *value* gives idempotency: the same times are never resubmitted to the
|
||||
same provider, but a manual correction (different times → different hash) submits the new
|
||||
value — which is exactly how TheIntroDB's weighted averaging expects corrections to arrive.
|
||||
|
||||
#### C5. Contribution service + triggers
|
||||
|
||||
A single `internal/markers/contribute.go` `ContributionService` funnels both triggers:
|
||||
`resolve enabled submitter providers (C6) → per provider: eligibility (C2) → idempotency check
|
||||
(hash vs marker_contributions) → Submitter.SubmitMarker → persist result`. Triggers:
|
||||
- **Manual / on-demand:** `POST /admin/files/{fileId}/contribute` (C7) submits a file's eligible
|
||||
markers to enabled providers (optionally scoped to one provider/segments).
|
||||
- **Auto local detections:** a daily `ContributeMarkersTask` (`internal/taskmanager/tasks/`)
|
||||
that, for each provider with `contribute_enabled && contribute_auto_local`, scans for episode
|
||||
files with `intro_markers_source = 'scanner'` and `intro_markers_confidence ≥` that provider's
|
||||
`contribute_min_confidence` lacking a current-hash `marker_contributions` row, and submits
|
||||
them rate/usage-limited. Runs after the existing 03:30 detection task.
|
||||
|
||||
#### C6. Per-provider configuration
|
||||
|
||||
Contribution and fetch behavior is **per provider**, so a single global toggle won't do.
|
||||
Credentials stay in `server_settings` (sensitive handling: `introdb.api_key`); behavioral
|
||||
config moves to a new table keyed by provider id:
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_provider_config.up.sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY, -- 'introdb'
|
||||
fetch_enabled boolean NOT NULL DEFAULT true, -- queried during reads (Layer B)
|
||||
fetch_priority integer NOT NULL DEFAULT 100, -- merge tiebreaker
|
||||
contribute_enabled boolean NOT NULL DEFAULT false, -- master submit switch (off)
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false, -- auto-submit local detections (off)
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- A provider's submissions require `contribute_enabled = true`. With it on, **manual**
|
||||
corrections submit on demand; **auto** submission of local detections additionally requires
|
||||
`contribute_auto_local = true`. Both contribute flags default off — "easily enabled with a
|
||||
setting," exactly one row per provider.
|
||||
- This same table carries the Layer B fetch enable/order, so "which providers do we read from"
|
||||
and "which do we submit to" live in one place. New providers insert a row (contribute off).
|
||||
- Surfaced and edited through the provider API (C7), so the future admin surface lists
|
||||
providers generically rather than hard-coding `introdb`.
|
||||
|
||||
The `markers.mode` / `markers.lazy_playback` globals and the `introdb.api_key` credential are
|
||||
unchanged.
|
||||
|
||||
#### C7. Admin provider + contribution API (fully fledged)
|
||||
|
||||
The complete backend surface, all under `RequireAdmin`, so a UI is purely additive later:
|
||||
|
||||
- `GET /admin/markers/providers` — list every registered provider: id, whether it is a
|
||||
`Submitter`, its `marker_provider_config` row, and (best-effort, cached) live `/user/stats`
|
||||
for submitter providers that have a key.
|
||||
- `PUT /admin/markers/providers/{provider}` — update that provider's config row
|
||||
(`fetch_enabled`, `fetch_priority`, `contribute_enabled`, `contribute_auto_local`,
|
||||
`contribute_min_confidence`). Validated; unknown provider → 404.
|
||||
- `POST /admin/markers/providers/{provider}/validate` — validate the configured key and return
|
||||
fresh `UserStats` (calls `Submitter.FetchUserStats`). Non-submitter → 400.
|
||||
- `POST /admin/files/{fileId}/contribute` — submit this file's eligible markers (C2) to enabled
|
||||
submitter providers; optional body `{provider?, segments?[]}` to scope to one provider /
|
||||
specific kinds. Returns per (provider, segment) `SubmissionResult`. Honors idempotency (C4):
|
||||
unchanged values are skipped, not resubmitted.
|
||||
- `GET /admin/files/{fileId}/contributions` — contribution history for the file from
|
||||
`marker_contributions` (per provider/segment: status, submitted_at, error).
|
||||
|
||||
Plus the manual marker endpoints from C3. Together these cover read, edit, submit, and audit
|
||||
without any web code.
|
||||
|
||||
## Data model summary
|
||||
|
||||
- **New table `marker_contributions`** (C4) — idempotent submission audit, keyed per
|
||||
(file, provider, segment, value-hash).
|
||||
- **New table `marker_provider_config`** (C6) — per-provider fetch + contribute behavior,
|
||||
seeded with `introdb` (fetch on, contribute off).
|
||||
- **No new flat server settings.** `introdb.api_key`, `markers.mode`, `markers.lazy_playback`
|
||||
are unchanged; behavioral provider config lives in `marker_provider_config`, not string keys.
|
||||
- **No new marker columns.** Per-segment confidence columns already exist; Layer A only changes
|
||||
how they're *populated* (per-segment instead of collapsed). `submission_count` stays
|
||||
transient (merge-ranking only) unless a later need appears.
|
||||
|
||||
## Client / cross-repo impact
|
||||
|
||||
- **No Android/Apple changes required.** This adds no new marker *kinds* and no new
|
||||
client-facing read contract: markers still flow to clients via the existing `WatchDetail`
|
||||
fields, the jellycompat `GET /MediaSegments` endpoint, and the `markers_updated` realtime
|
||||
event. Manual editing and contribution are server-side (admin API); their eventual UI is
|
||||
deferred web-admin work.
|
||||
- **Plugin SDK:** untouched. `marker_provider.v1` is a deferred future capability (Layer B
|
||||
seam), not part of this work.
|
||||
- A future *per-user* contribution model (each user links their own TheIntroDB account) *would*
|
||||
be client work (a "Connect TheIntroDB" screen) — explicitly deferred.
|
||||
|
||||
## Build sequence
|
||||
|
||||
Each phase is shippable on its own; later phases depend only on earlier ones. Every phase is
|
||||
**backend-only** — no web UI.
|
||||
|
||||
1. **Phase 1 — Correctness foundation (Layer A).** TVDB lookups; capture
|
||||
confidence/submission_count; best-candidate selection; per-segment confidence in the
|
||||
payload. *Directly answers "are we doing this properly," low risk, no new surface.*
|
||||
2. **Phase 2 — Multi-source dispatch + provider config (Layer B + C6).** `Submitter` interface;
|
||||
`FetchMerged` query-all/best-wins; `marker_provider_config` table + repo driving fetch
|
||||
enable/order. *No behavior change with only TheIntroDB enabled.*
|
||||
3. **Phase 3 — Submission client + tracking + service (C1, C4, C5 core).** `SubmitSegment` /
|
||||
`FetchUserStats`; `marker_contributions` migration + repo; `ContributionService` (eligibility,
|
||||
idempotency, per-provider gating).
|
||||
4. **Phase 4 — Manual marker API (C3) + provider/contribution API (C7).** File- and item-level
|
||||
manual read/write/clear; provider list/config/validate; per-file contribute + history. The
|
||||
full admin contract, no UI.
|
||||
5. **Phase 5 — Auto-contribution (C5 task).** `ContributeMarkersTask`: per provider, when
|
||||
`contribute_enabled && contribute_auto_local`, submit episode intro markers above the
|
||||
provider's `contribute_min_confidence`, usage-limit aware, idempotent.
|
||||
|
||||
Deferred (separate UI-only follow-up): web admin surfaces over C3/C6/C7 — a marker editor in
|
||||
the player chrome and a provider/contribution panel in admin settings.
|
||||
|
||||
## Risks & trade-offs
|
||||
|
||||
- **Data-quality / pollution.** Auto-submitting chromaprint detections (confidence 0.65–0.90)
|
||||
risks polluting a provider's data. Mitigation: `contribute_enabled` and `contribute_auto_local`
|
||||
default **off** per provider, and the default `contribute_min_confidence` of 0.95 admits only
|
||||
chapter / chapter+silence detections even once enabled. Manual, high-trust corrections are the
|
||||
intended first contribution. Revisit chromaprint auto-submission after watching acceptance
|
||||
rates via `/user/stats`.
|
||||
- **Single-account weighting.** All contributions credit one server account; TheIntroDB's
|
||||
weighting may treat a single account's mass submissions differently than many users'. Note
|
||||
the per-user seam; acceptable for v1.
|
||||
- **Circular contribution.** Never contribute `source = online`. Enforced in C2 eligibility and
|
||||
worth a unit test.
|
||||
- **Usage limits.** Bulk auto-submission can hit the submission usage limit; the task must obey
|
||||
`X-UsageLimit-Reset` and resume next run. Idempotency means an interrupted run safely resumes.
|
||||
- **"Query all" cost.** `FetchMerged` issues one lookup per fetch-enabled provider; with only
|
||||
TheIntroDB it's identical to today, and per-provider caching bounds the cost as sources are
|
||||
added.
|
||||
- **Movies & non-intro segments** have no local detector, so their only contribution route is
|
||||
manual correction — acceptable and explicit.
|
||||
|
||||
## Open questions for review
|
||||
|
||||
1. **Contribute to all enabled providers, or require explicit per-call selection?** The design
|
||||
defaults `POST /admin/files/{id}/contribute` to "every enabled submitter," with an optional
|
||||
`provider` scope. Acceptable, or should submission always name its target provider?
|
||||
2. **Manual-clear behavior.** Clearing a manual segment removes the manual value and lets a
|
||||
later detection/fetch repopulate it (we can't restore the exact pre-edit value). Is
|
||||
"remove, then re-detect" the right contract, or do you want a marker history/undo stack?
|
||||
3. **Per-provider config home.** This design uses a `marker_provider_config` table (clean for N
|
||||
providers, future per-library override) rather than flat `server_settings` keys. Confirm the
|
||||
table is the direction you want.
|
||||
|
||||
## File-level blueprint
|
||||
|
||||
**Backend (Go) — Layer A**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `tvdb_id` query support; cache-key extension.
|
||||
- `internal/markers/introdb/types.go` — *modify*: `confidence`/`submission_count` fields.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: read TVDB; real confidence; best-candidate pick.
|
||||
- `internal/markers/types.go` — *modify*: per-segment confidence in `Marker`/Result plumbing.
|
||||
- `internal/markers/write.go` — *modify*: per-segment confidence in `MarkerUpdatePayload` + `BuildUpdatePayload`.
|
||||
- `internal/scanner/file_repo.go` — *modify*: map per-segment confidence in `UpsertMarkers`.
|
||||
|
||||
**Backend (Go) — Layer B + per-provider config**
|
||||
- `internal/markers/types.go` — *modify*: `Submitter` interface; `SubmissionRequest`/`Result`, `UserStats`; `Registry.FetchMerged`.
|
||||
- `internal/markers/provider_config.go` — *create*: `marker_provider_config` repo (load/update; drives fetch enable/order + contribute gating).
|
||||
- `migrations/<next>_marker_provider_config.{up,down}.sql` — *create*.
|
||||
|
||||
**Backend (Go) — Layer C**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `SubmitSegment` (`POST /v3/submit`), `FetchUserStats` (`GET /v3/user/stats`), usage-limit handling.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: implement `markers.Submitter`; drop the "submissions not supported" note in `types.go`.
|
||||
- `internal/markers/contribute.go` — *create*: `ContributionService` (eligibility, idempotency, per-provider gating, submit, persist).
|
||||
- `internal/markers/contribution_repo.go` — *create*: `marker_contributions` CRUD + value-hash check.
|
||||
- `migrations/<next>_marker_contributions.{up,down}.sql` — *create*.
|
||||
- `internal/api/handlers/admin_markers.go` — *create*: manual marker read/write/clear (C3) + per-file contribute + history (C7 file endpoints).
|
||||
- `internal/api/handlers/admin_marker_providers.go` — *create*: provider list/config/validate (C7 provider endpoints).
|
||||
- `internal/api/router.go` — *modify*: register the new `RequireAdmin` routes.
|
||||
- `internal/taskmanager/tasks/contribute_markers.go` — *create*: daily auto-contribution task.
|
||||
- `cmd/silo/main.go` — *modify*: wire `ContributionService`, provider-config repo, the task, and the `Submitter`.
|
||||
|
||||
**Frontend (React/TS) — deferred (not this round)**
|
||||
- Marker editor over `PUT /admin/files/{id}/markers`; provider/contribution panel over the C7
|
||||
endpoints; `IntroDBCredentialCard` "validate key & show stats". Listed for traceability only.
|
||||
|
||||
**Reference (read-only, not in this repo)**
|
||||
- TheIntroDB v3 contract verified against `github.com/TheIntroDB/theintrodb-npm`
|
||||
(`src/funcs.ts`, `src/types.ts`, `docs/`) and `github.com/TheIntroDB/jellyfin-plugin`
|
||||
(`TheIntroDB/Api/*.cs`).
|
||||
@@ -0,0 +1,311 @@
|
||||
# AI services: shared core, metadata translation, Whisper ASR — design
|
||||
|
||||
**Date:** 2026-06-10
|
||||
**Status:** Draft, pending review
|
||||
**Scope:** Extract the AI subtitle translation engine into a shared AI core, add AI translation of movie/series/season/episode descriptions into the existing localization tables, and add Whisper ASR subtitle generation (`transcribe`, `transcribe_translate`). One PR.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
`internal/subtitles/ai` ships an on-demand LLM subtitle translator. The LLM plumbing inside
|
||||
it (OpenAI-compatible chat client with retry/backoff, batched indexed-JSON translation,
|
||||
job lifecycle with idempotency/heartbeat/reaper/bounded concurrency) is generic, but it is
|
||||
buried in a subtitle-specific package, so nothing else can use it.
|
||||
|
||||
Two features want exactly that plumbing:
|
||||
|
||||
1. **Metadata translation.** Libraries whose `metadata_language` the providers cannot fully
|
||||
serve end up with overviews/taglines in the item's default language (usually English).
|
||||
The localization tables (`media_item_localizations`, `season_localizations`,
|
||||
`episode_localizations`) and the serving path (`LocalizeItemModel` and friends in
|
||||
`internal/catalog/detail.go`) already exist — the gap is producing rows for languages the
|
||||
providers don't have.
|
||||
2. **Whisper ASR.** `subtitle_ai_jobs.kind` already reserves `transcribe` and
|
||||
`transcribe_translate`; media with no usable text subtitle track (foreign audio with only
|
||||
bitmap subs, or no subs at all) currently has no AI path to a subtitle track.
|
||||
|
||||
## Goals
|
||||
|
||||
- One shared AI core: a single OpenAI-compatible endpoint configuration (chat + audio
|
||||
transcription), one retry/backoff implementation, one generic batched text translator,
|
||||
one job-lifecycle runner — consumed by subtitle translation, metadata translation, and ASR.
|
||||
- AI-translated overviews and taglines stored in the **existing** localization tables, so
|
||||
web, Android, Apple, and jellycompat clients get them through the current serving path
|
||||
with **zero client changes**.
|
||||
- Provenance on AI-writable localization fields so refreshes never regress data quality:
|
||||
`manual` beats `provider` beats `ai`. A later provider localization overwrites an AI
|
||||
translation; the reverse never happens.
|
||||
- Two metadata-translation triggers: a manual admin action, and an opt-in per-library
|
||||
fallback ("provider had no localization for the library language → AI-translate") wired
|
||||
into metadata ingestion.
|
||||
- ASR jobs that produce ordinary `downloaded_subtitles` rows (provider `transcribed`),
|
||||
with `transcribe_translate` chaining the existing LLM translator for a target-language
|
||||
track; live cue streaming to the requesting playback session, same as translation today.
|
||||
- Total AI load on the configured endpoint bounded by **one** shared semaphore.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **Episode/movie/series title translation.** Titles often have official localized forms
|
||||
that are not literal translations; provider data only.
|
||||
- **Collection names or descriptions.** Deferred: collections have no localization table
|
||||
yet. The metadata translation service is written against content-ID + field-list inputs,
|
||||
so a `collection_localizations` table can be added in a follow-up without reshaping it.
|
||||
- **Genre translation.** If ever wanted, that is a static mapping, not an LLM call.
|
||||
- **Per-profile metadata language.** Presentation language stays library-scoped
|
||||
(`media_folders.metadata_language`); changing that is an orthogonal feature.
|
||||
- **A localization editor UI.** The `manual` provenance value is honored by the write
|
||||
rules but no UI sets it yet (the existing metadata editor edits base fields, not
|
||||
localizations).
|
||||
- **Whisper word-level timing, diarization, VAD-aligned chunking.** v1 uses fixed-length
|
||||
chunks; boundary-word artifacts are an accepted limitation noted for follow-up.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Package layout
|
||||
|
||||
```
|
||||
internal/ai/llm Client: OpenAI-compatible /v1/chat/completions (moved from
|
||||
internal/subtitles/ai/client.go) + new /v1/audio/transcriptions.
|
||||
Shared retry/backoff/429 handling. Config for the endpoint(s).
|
||||
internal/ai/translate Generic batched text translation: []Segment{ID, Text} in,
|
||||
translated segments out. Indexed-JSON protocol, context
|
||||
neighbors, malformed-response retries — moved from
|
||||
internal/subtitles/ai/translator.go and de-subtitled.
|
||||
internal/ai/jobrunner Generic job lifecycle: bounded dispatch off a shared semaphore,
|
||||
heartbeat loop, stale-job reaper, cancel registry — extracted from
|
||||
internal/subtitles/ai/service.go behind a small repo interface.
|
||||
internal/subtitles/ai Stays: subtitle job semantics. Translate path becomes a thin
|
||||
adapter (cues ↔ segments); gains the transcribe paths.
|
||||
internal/metadata/translation
|
||||
New: metadata translation job service + repo + provenance-aware
|
||||
writes into the localization repos.
|
||||
```
|
||||
|
||||
Package names: `llm`, `aitranslate`, `jobrunner` (import paths above; `aitranslate` avoids
|
||||
a name collision when imported from package `ai`).
|
||||
|
||||
### Shared semaphore
|
||||
|
||||
`cmd/silo` builds one `chan struct{}` of size `ai.max_concurrent_jobs` and hands it to both
|
||||
job services via `jobrunner`. Subtitle, metadata, and ASR jobs all draw from it, so the
|
||||
operator's endpoint sees a bounded number of concurrent jobs regardless of job mix.
|
||||
(An ASR job holds one slot for its whole pipeline: extract → transcribe → translate.)
|
||||
|
||||
## Settings
|
||||
|
||||
Connection settings move to a shared `ai.*` namespace. Because encrypted values are
|
||||
GCM-bound to their setting key (see `internal/catalog/encrypted_settings_repo.go`), the
|
||||
legacy rows are **not renamed**; the loader falls back to them, following the precedent of
|
||||
`recommendations.embedding_auth_token` → `recommendations.openai_api_key` in
|
||||
`internal/config/db_loader.go`.
|
||||
|
||||
| Key | Default | Fallback | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `ai.base_url` | `https://api.openai.com` | `subtitle_ai.base_url` | |
|
||||
| `ai.api_key` | `""` | `subtitle_ai.api_key` | sensitive |
|
||||
| `ai.chat_model` | `gpt-4o-mini` | `subtitle_ai.chat_model` | |
|
||||
| `ai.max_concurrent_jobs` | `2` | `subtitle_ai.max_concurrent_jobs` | shared semaphore size |
|
||||
| `ai.asr_model` | `whisper-1` | — | |
|
||||
| `ai.asr_base_url` | `""` (= `ai.base_url`) | — | separate local Whisper server |
|
||||
| `ai.asr_api_key` | `""` (= `ai.api_key`) | — | sensitive |
|
||||
| `subtitle_ai.enabled` | `false` | — | existing; subtitle translation toggle |
|
||||
| `subtitle_ai.transcribe_enabled` | `false` | — | new; ASR toggle |
|
||||
| `subtitle_ai.batch_size` | `40` | — | existing; cues per chat request |
|
||||
| `subtitle_ai.context_neighbors` | `2` | — | existing |
|
||||
| `metadata_ai.enabled` | `false` | — | new; metadata translation toggle |
|
||||
|
||||
`ai.api_key` and `ai.asr_api_key` join `sensitiveSettingKeys`. The admin UI reads through
|
||||
the same fallback (effective values come from the loaded config) and writes the new keys.
|
||||
|
||||
Config structs: `AIConfig` (connection + concurrency), `SubtitleAIConfig` (slims down to
|
||||
toggles + batch tuning), new `MetadataAIConfig`. Readiness: subtitle translate requires
|
||||
`subtitle_ai.enabled` + chat config; ASR requires `subtitle_ai.transcribe_enabled` + ASR
|
||||
config; metadata requires `metadata_ai.enabled` + chat config.
|
||||
|
||||
## Metadata translation
|
||||
|
||||
### What gets translated
|
||||
|
||||
| Target | Fields |
|
||||
| --- | --- |
|
||||
| movie / series (`media_item_localizations`) | `overview`, `tagline` |
|
||||
| season (`season_localizations`) | `overview` |
|
||||
| episode (`episode_localizations`) | `overview` |
|
||||
|
||||
Titles, sort titles, and artwork columns in those rows are never written by AI.
|
||||
|
||||
### Provenance
|
||||
|
||||
New columns (default `'provider'`, check-constrained to `provider|ai|manual`):
|
||||
|
||||
- `media_item_localizations.overview_source`, `media_item_localizations.tagline_source`
|
||||
- `season_localizations.overview_source`
|
||||
- `episode_localizations.overview_source`
|
||||
|
||||
Write rules, enforced in the localization repos (single-statement upserts with CASE):
|
||||
|
||||
| Writer | May overwrite | Sets source | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| provider ingestion | `provider`, `ai` | `provider` | a non-empty provider value always replaces an AI value; an **empty** provider value never blanks an existing one |
|
||||
| AI translation | `ai`, empty | `ai` | with `force`, may also overwrite `provider` (admin explicitly re-translating) — never `manual` |
|
||||
| manual (future editor) | anything | `manual` | |
|
||||
|
||||
Serving hardening: `LocalizeItemModel` / `LocalizeSeasonModel` / `LocalizeEpisodeModel`
|
||||
must only override base fields with **non-empty** localization values, since AI-created
|
||||
rows legitimately carry empty titles. (Verify current behavior; fix if it clobbers.)
|
||||
|
||||
### Job model
|
||||
|
||||
New table `metadata_translation_jobs`, mirroring `subtitle_ai_jobs` conventions
|
||||
(status enum, progress, idempotency partial-unique index, heartbeat):
|
||||
|
||||
- `target_kind` (`item` | `season` | `episode`), `content_id`, `include_children`
|
||||
- `target_language`, `source_language` (default language of the base row)
|
||||
- `engine`, `model`, `status`, `progress`, `progress_message`, `fields_done`, `fields_total`
|
||||
- `error_message`, `idempotency_key`, `requested_by`, `force`, timestamps, `heartbeat_at`
|
||||
|
||||
A job for a series item with `include_children` expands to: item overview + tagline, every
|
||||
season overview, every episode overview. All fields become segments
|
||||
(`item:overview`, `season:<content_id>:overview`, `episode:<content_id>:overview`, …) fed
|
||||
to the generic translator in batches (metadata batch size constant, smaller than the
|
||||
subtitle one — paragraphs, not cue lines), so episode descriptions share terminology in one
|
||||
prompt context. The system prompt names the series/movie and year for grounding.
|
||||
|
||||
Skip logic in the run loop (not at enqueue): a field whose localization value is already
|
||||
non-empty is skipped unless `force` — so the auto trigger costs zero LLM calls on repeat
|
||||
refreshes. If every field is skipped the job completes immediately.
|
||||
|
||||
Idempotency key: `content_id | target_kind | target_language | model`, active-only partial
|
||||
unique index (completed/failed rows never block a retry).
|
||||
|
||||
### Triggers
|
||||
|
||||
1. **Manual.** Admin action on the item detail metadata editor: pick target language
|
||||
(defaults to the library's `metadata_language`), `include_children`, `force`. Gated by
|
||||
the same permission as metadata editing.
|
||||
2. **Auto fallback.** New `media_folders.auto_translate_metadata boolean NOT NULL DEFAULT
|
||||
false`. At the end of a metadata refresh for an item (after provider localizations are
|
||||
persisted in `internal/metadata/service.go`), if the folder has the flag, has a
|
||||
`metadata_language`, that language differs from the item's default metadata language,
|
||||
and the localization row for it is missing a translated field — enqueue a non-force job
|
||||
for the item (children included for series). Enqueue is fire-and-forget; refresh latency
|
||||
is unaffected.
|
||||
|
||||
### API
|
||||
|
||||
Admin-gated, mirroring the subtitle AI handler shapes:
|
||||
|
||||
- `GET /api/v1/metadata/ai/status`
|
||||
- `POST /api/v1/metadata/ai/translate` `{content_id, target_kind, target_language, include_children, force}`
|
||||
- `GET /api/v1/metadata/ai/jobs/{job_id}` / `POST …/cancel`
|
||||
- `GET /api/v1/metadata/ai/jobs?content_id=…`
|
||||
|
||||
Progress UI polls the job endpoint (jobs are short; no websocket events in v1).
|
||||
|
||||
## Whisper ASR
|
||||
|
||||
### Kinds
|
||||
|
||||
`subtitle_ai_jobs.kind` gains its reserved values:
|
||||
|
||||
- `transcribe` — audio track → subtitle track in the spoken language.
|
||||
- `transcribe_translate` — `transcribe`, then the existing LLM translation chained to
|
||||
`target_language`. Both the transcript track and the translated track are stored (the
|
||||
transcript is a cache: a second target language skips ASR entirely because the transcript
|
||||
is now a selectable text source for a plain `translate` job). `result_subtitle_id`
|
||||
points at the translated track.
|
||||
|
||||
For ASR kinds, `source_index` holds the **audio** track index (`-1` = default/first audio
|
||||
track); `source_language` is an optional hint passed to Whisper, else taken from the
|
||||
detected `verbose_json.language`.
|
||||
|
||||
### Pipeline
|
||||
|
||||
1. Resolve the audio track from `media_files`; reject files with no audio.
|
||||
2. Extract once with ffmpeg into fixed chunks in a job temp dir
|
||||
(`-vn -map 0:a:<idx> -ac 1 -ar 16000 -c:a pcm_s16le -f segment -segment_time 600`):
|
||||
16 kHz mono WAV, 10-minute chunks (~19 MB, under typical 25 MB API limits). Temp dir
|
||||
removed on every exit path.
|
||||
3. Per chunk: `POST /v1/audio/transcriptions` (multipart: file, model,
|
||||
`response_format=verbose_json`, optional `language`, `temperature=0`) via the shared
|
||||
`llm.Client` with the shared retry/backoff. Segment timestamps are offset by the chunk
|
||||
start and merged.
|
||||
4. Cue building: one cue per Whisper segment; text wrapped to at most two lines
|
||||
(~42 chars/line). Empty/whitespace segments dropped.
|
||||
5. Store as `downloaded_subtitles` (provider `transcribed`, SRT, release name
|
||||
`"<Language> (AI transcribed)"`), notify `SubtitleReady`.
|
||||
6. `transcribe_translate`: feed the cues to the existing `Translator`; store the translated
|
||||
track (provider `translated`) and complete.
|
||||
|
||||
Chunks are processed starting from the chunk containing the viewer's playhead, then
|
||||
wrapping — same UX as translation — and cues stream live to the requesting session through
|
||||
the existing notifier. Progress bands: extract 0–10 %, transcribe 10–70 % (per chunk),
|
||||
translate 70–95 %.
|
||||
|
||||
Known v1 limitation: fixed chunk boundaries can clip a word at the seam; acceptable, noted
|
||||
for a silence-aligned follow-up.
|
||||
|
||||
### API / UI
|
||||
|
||||
The existing enqueue endpoint `POST /api/v1/subtitles/ai/translate` gains an optional
|
||||
`kind` (default `translate`; backward compatible). `GET /api/v1/subtitles/ai/status` gains
|
||||
`transcribe_enabled`. The player's translate modal gains a "Generate from audio" mode that
|
||||
lists audio tracks and a target language — the natural path when the only subtitle tracks
|
||||
are bitmap (today's hard error) or none exist.
|
||||
|
||||
## Frontend
|
||||
|
||||
- **AI Services settings page** (`web/src/pages/admin-settings/`): connection card
|
||||
(base URL, API key, chat model, ASR model, optional ASR base URL/key), features card
|
||||
(subtitle translation, ASR, metadata translation toggles; concurrency; batch tuning).
|
||||
The AI card currently inside `SubtitlesSettings.tsx` moves here; subtitle settings keep
|
||||
the download-provider config.
|
||||
- **Metadata editor**: "Translate with AI" action in `EditMetadataDialog.tsx` (language,
|
||||
include children, force), with job-poll progress and query invalidation on completion.
|
||||
- **Library settings**: `auto_translate_metadata` toggle next to the metadata-language
|
||||
field.
|
||||
- **Player**: "Generate from audio" mode in `SubtitleTranslateModal.tsx`.
|
||||
|
||||
## Client (Android / Apple) impact
|
||||
|
||||
None required. Translated overviews/taglines arrive through the existing localized detail
|
||||
responses; ASR/translated tracks arrive as ordinary downloaded subtitles. Optional
|
||||
follow-ups in the client repos: surface "Generate from audio" in their subtitle menus
|
||||
(the jobs API is shared) and the library auto-translate toggle in their admin surfaces.
|
||||
|
||||
## Reliability
|
||||
|
||||
- All jobs: idempotency dedup of in-flight work, heartbeat + stale reaper (crash-safe,
|
||||
multi-instance safe), terminal-state handling on cancel/shutdown — one implementation in
|
||||
`jobrunner`, no copy-paste between the two services.
|
||||
- Provider-wins/manual-wins provenance is enforced in SQL (single-statement upserts), not
|
||||
in racy read-modify-write Go code.
|
||||
- The shared semaphore bounds total endpoint load; ASR temp space is cleaned on success,
|
||||
failure, cancel, and shutdown.
|
||||
- Auto-translate re-runs are free: skip-if-filled happens before any LLM call.
|
||||
|
||||
## Migrations
|
||||
|
||||
One Goose migration (`make migrate-create NAME=ai_metadata_translation_and_asr`):
|
||||
|
||||
1. `metadata_translation_jobs` table + active-idempotency / content / status indexes.
|
||||
2. Provenance columns on the three localization tables (+ check constraints).
|
||||
3. `media_folders.auto_translate_metadata`.
|
||||
|
||||
Down: drop the table, columns, and constraint.
|
||||
|
||||
## Risks / open questions
|
||||
|
||||
- **Whisper-endpoint compatibility.** OpenAI, Groq, faster-whisper-server/speaches all
|
||||
speak `verbose_json`, but segment quality varies; the cue builder must tolerate missing
|
||||
segment arrays (fall back to one cue per chunk worth of `text` is *not* acceptable —
|
||||
fail the job with a clear message instead).
|
||||
- **`LocalizeItemModel` empty-field semantics** must be verified before AI rows (empty
|
||||
titles) ship; this is called out as an explicit plan task.
|
||||
- **Long-running ASR on slow local Whisper servers** can exceed the 10-minute HTTP client
|
||||
timeout per chunk; the transcription call uses a per-request timeout sized for chunk
|
||||
duration rather than the client default.
|
||||
- **Settings sprawl.** The `ai.*`/legacy fallback leaves old `subtitle_ai.*` connection
|
||||
rows in place indefinitely; harmless, but a future settings-GC could prune them once the
|
||||
new keys are written.
|
||||
@@ -0,0 +1,188 @@
|
||||
# Core Notifications System — Design
|
||||
|
||||
Date: 2026-06-10
|
||||
Status: approved (design); implementation plan to follow
|
||||
|
||||
## Summary
|
||||
|
||||
A first-party, whole-system notification layer for Silo: persisted per-user notifications with an inbox, read state, per-category preferences, real-time toasts over the existing event WebSocket, and admin announcements. Ships across the server, web UI, and both mobile apps (in-app only — no APNs/FCM push in this project). External delivery (Discord, email, webhooks) remains the job of the existing `silo.notifications` plugin, which consumes the same events.
|
||||
|
||||
## Goals
|
||||
|
||||
- Persisted per-user notifications: inbox, unread badge, read/dismiss state.
|
||||
- Real-time in-app delivery to connected clients via the existing `events.Hub` / `/api/v1/events/ws` pipeline.
|
||||
- Five categories: `request`, `content`, `announcement`, `system`, `admin` (operational alerts for admin users), plus opt-in `content_digest`.
|
||||
- Per-category user preferences (announcements not mutable).
|
||||
- Admin announcements with audience targeting and expiry.
|
||||
- A documented `notifications.send` event contract so any plugin can create in-app notifications (and, via the existing external plugin's subscription to the same event, external ones).
|
||||
- Core request lifecycle events (`request.*`) — net-new publishing from `internal/requests`, consumed by notifications and available to the requests UI.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Mobile push (APNs/FCM) — future project; nothing here blocks it.
|
||||
- Request-system unification (`request_router.v1`, migrating ebook/audiobook request plugins onto `media_requests`) — separate future project. When it lands, the book plugins' `notifications.send` publishes are replaced by core `request.*` events with no notifications schema/API change.
|
||||
- Absorbing external delivery channels into core.
|
||||
- Fine-grained per-show/per-author subscription rules (watchlist/favorites are the v1 signal).
|
||||
|
||||
## Architecture
|
||||
|
||||
New package `internal/notifications` in silo-server:
|
||||
|
||||
1. **Materializer** — in-process consumer registered on `events.Hub` at startup (same pattern as `internal/plugins/event_dispatcher.go`). Runs events through per-category matchers; writes `notifications` rows for each recipient.
|
||||
2. **Store** — Postgres repository: insert (idempotent), list (cursor), unread count, mark read, dismiss, preferences CRUD.
|
||||
3. **Notifier** — after insert, publishes `notification.created` on a new `ChannelNotifications` event channel, user-scoped like `ChannelUserState`. The WS handler (`internal/api/handlers/events_ws.go`) adds this channel to the per-user subscribe set; frames carry the full notification object. Subscribe-time snapshot carries unread count only.
|
||||
4. **API handlers** — REST endpoints below.
|
||||
5. **Service API** — `notifications.Create(...)` for direct in-process callers (system/security events).
|
||||
|
||||
Creation is DB-first and synchronous: a notification exists when the row commits. WS fan-out is best-effort; clients that miss frames reconcile on next fetch. No delivery queue or retry in core — "delivery" is reading your own table.
|
||||
|
||||
## Data model
|
||||
|
||||
One migration, three tables:
|
||||
|
||||
```sql
|
||||
notifications (
|
||||
id bigserial PRIMARY KEY,
|
||||
user_id int NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
profile_id text NULL REFERENCES user_profiles(id) ON DELETE CASCADE, -- NULL = user-wide
|
||||
category text NOT NULL, -- 'request' | 'content' | 'announcement' | 'system' | 'admin'
|
||||
type text NOT NULL, -- e.g. 'request.approved', 'content.watchlist_episode'
|
||||
title text NOT NULL,
|
||||
body text NOT NULL,
|
||||
link text NULL, -- in-app deep link
|
||||
item_id bigint NULL, -- optional content reference (artwork, dedup)
|
||||
source_event text NULL, -- originating event name (audit)
|
||||
dedup_ref text NULL, -- idempotency: UNIQUE (user_id, type, dedup_ref) WHERE dedup_ref IS NOT NULL
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
read_at timestamptz NULL,
|
||||
dismissed_at timestamptz NULL,
|
||||
expires_at timestamptz NULL
|
||||
);
|
||||
-- index: (user_id, created_at DESC) partial WHERE dismissed_at IS NULL
|
||||
-- partial unique index for dedup as above; inserts use ON CONFLICT DO NOTHING
|
||||
|
||||
notification_preferences (
|
||||
user_id int NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
category text NOT NULL, -- 'request' | 'content' | 'system' | 'admin' | 'content_digest'
|
||||
enabled bool NOT NULL,
|
||||
PRIMARY KEY (user_id, category)
|
||||
);
|
||||
-- absent row = enabled, except content_digest which defaults OFF
|
||||
-- 'announcement' is not a valid category here (not mutable)
|
||||
|
||||
announcements (
|
||||
id bigserial PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
body text NOT NULL,
|
||||
audience jsonb NOT NULL, -- {"all":true} | {"user_ids":[...]} | {"library_ids":[...]}
|
||||
created_by int REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NULL
|
||||
);
|
||||
```
|
||||
|
||||
Announcements fan out to `notifications` rows at publish time (audience resolved then), so the inbox is always a single-table query and read state is per-user.
|
||||
|
||||
Child profiles: rows with `category IN ('request','system','admin')` are filtered out of child-profile sessions server-side; content matching never targets a child profile unless the matched watchlist/favorite belongs to that profile.
|
||||
|
||||
Retention (nightly job, existing scheduler pattern): dismissed rows purged after 30 days; all rows after 90 days; unread rows from expired announcements at expiry.
|
||||
|
||||
## API
|
||||
|
||||
User endpoints (authenticated, profile from claims):
|
||||
|
||||
```
|
||||
GET /api/v1/notifications ?unread=1&category=&cursor=&limit=
|
||||
GET /api/v1/notifications/unread-count → {count}
|
||||
POST /api/v1/notifications/read {ids:[...]} | {all:true}
|
||||
POST /api/v1/notifications/{id}/dismiss
|
||||
GET /api/v1/notifications/preferences
|
||||
PUT /api/v1/notifications/preferences {category: enabled, ...}
|
||||
```
|
||||
|
||||
List returns user-wide rows plus active-profile rows, newest first, cursor-paginated in the standard envelope. Child-profile filtering server-side.
|
||||
|
||||
Admin endpoints (RequireAdmin group):
|
||||
|
||||
```
|
||||
GET /api/v1/admin/announcements
|
||||
POST /api/v1/admin/announcements {title, body, audience, expires_at?}
|
||||
DELETE /api/v1/admin/announcements/{id} -- also dismisses unread fanned-out rows
|
||||
```
|
||||
|
||||
## Events
|
||||
|
||||
### New: core request lifecycle events
|
||||
|
||||
`internal/requests/service.go` and the reconciliation task publish on a new `ChannelRequests`:
|
||||
|
||||
- `request.submitted`, `request.approved`, `request.declined`, `request.completed`, `request.failed`
|
||||
- Payload: `request_id, user_id, profile_id, title, media_type, quality, status`
|
||||
|
||||
`ChannelRequests` is added to WS subscribe sets (requesting user + admins), giving the requests UI live updates as a side benefit. This is the only change to the requests subsystem in this project.
|
||||
|
||||
### New: `notifications.send` plugin contract
|
||||
|
||||
The materializer consumes `notifications.send` events:
|
||||
|
||||
```json
|
||||
{ "user_id": 1, "profile_id": null, "category": "request",
|
||||
"type": "request.fulfilled", "title": "...", "body": "...",
|
||||
"link": null, "dedup_ref": "ebook-req-123:fulfilled" }
|
||||
```
|
||||
|
||||
Payloads are validated; malformed events are dropped with a logged warning. The external `silo.notifications` plugin already subscribes to this event name, so one publish reaches both in-app and external channels. The ebook/audiobook request plugins each add one `PublishEvent("notifications.send", ...)` at their acknowledged/fulfilled/failed transitions (their existing status events do not carry `user_id`; the plugins own that context).
|
||||
|
||||
### Matchers
|
||||
|
||||
- **request** — consumes core `request.*` events only. Dedup ref `(request_id, status)`.
|
||||
- **content** — consumes `library.item_added` / `catalog.library.changed`. For each added item: resolve series/author; match per profile against watchlist, favorites (author/series link), and in-progress series (last play within 21 days); union, dedup, filter by library access. Burst guard: one notification per series per profile per hour (season imports collapse).
|
||||
- **content_digest** — nightly job (not event-driven), opt-in: one row per user summarizing the previous day's additions to accessible libraries.
|
||||
- **system** — not matched from the hub; emitted via `notifications.Create()` directly at the source (password change, new-device login, guest-pass redemption). Sites also publish their events as today; the explicit call means security notices never depend on matcher wiring.
|
||||
- **announcement** — fan-out at publish time.
|
||||
- **admin** — consumes existing `ChannelJobs` / `ChannelScans` / `ChannelTasks` failure events plus plugin-host crash/install failures; targets admin users; repeat-failure throttle (same source, max one per hour).
|
||||
|
||||
Preference checks run in the materializer before insert (cached per-user prefs lookup).
|
||||
|
||||
## Web UI
|
||||
|
||||
1. **Bell + dropdown** in the top bar: unread badge from `unread-count`, kept live by `ChannelNotifications` frames via the existing `RealtimeEventsProvider` socket. Dropdown: latest ~10, mark-all-read, deep links.
|
||||
2. **Toasts** — WS notification frames fire sonner toasts; suppressed while the dropdown is open or when the frame targets another profile.
|
||||
3. **Inbox page** (`/notifications`) — paginated list, category filter tabs, dismiss, mark-read on view.
|
||||
4. **Settings** — per-category toggles (`content_digest` default off; `admin` visible to admins only). Admin: Announcements page (list/compose/expire) under admin settings.
|
||||
|
||||
## Mobile (in-app, both platforms)
|
||||
|
||||
Same three features in silo-apple and silo-android against the frozen REST API: inbox screen (list, mark-read, dismiss), unread badge on existing navigation, preferences screen. Transport: REST refresh of unread-count + first page on app foreground and pull-to-refresh. No WS subscription in v1 (current app sockets are playback-scoped; no background delivery exists without push anyway). TV variants: badge + read-only inbox, no preferences editing.
|
||||
|
||||
## Error handling
|
||||
|
||||
- Matcher isolation: a panic/error in one matcher logs and skips that event for that matcher only; never blocks the hub or other matchers (same containment as the plugin event dispatcher).
|
||||
- Idempotent inserts via the dedup partial unique index + `ON CONFLICT DO NOTHING`; replayed events cannot double-notify.
|
||||
- WS fan-out is fire-and-forget; DB rows are the source of truth.
|
||||
- `notifications.send` validation failures: drop + warn (plugin bugs must not break core paths).
|
||||
|
||||
## Testing
|
||||
|
||||
- Unit tests per matcher with synthetic events; content-matcher edge cases: child profiles, library access, burst guard, dedup.
|
||||
- Store tests against Postgres (repo convention).
|
||||
- One integration test: event → materializer → row → WS frame through a real hub.
|
||||
- API handler tests: auth, profile scoping, child filtering, admin gates.
|
||||
- Web: component tests for bell/inbox; extend the RealtimeEventsProvider test pattern for the new channel.
|
||||
|
||||
## Implementation sequencing
|
||||
|
||||
1. **Server** — migration, store, materializer + matchers, request event publishing, API, WS channel, retention job.
|
||||
2. **Web** — bell, toasts, inbox, settings, announcements admin.
|
||||
3. **Mobile** — apple + android in-app surfaces against the frozen API.
|
||||
4. (Separate projects, later: request unification via `request_router.v1`; mobile push.)
|
||||
|
||||
## Future work
|
||||
|
||||
- Request unification: core `request.*` events replace the book plugins' `notifications.send` publishes; notifications unchanged.
|
||||
- Push: APNs/FCM device registry; the notifier gains a push fan-out alongside WS.
|
||||
- Digest scheduling preferences (weekly, time-of-day).
|
||||
- Profile claims enrichment: embed `is_child` in profile-scoped auth claims. Fixes two implementation-time findings: the WS pipeline cannot child-filter (snapshot and live frames use client-asserted profile identity; REST remains authoritative and fails closed — see the snapshot comment in `internal/api/handlers/events_ws.go`), and the REST `childSafe` check costs one profile lookup per request.
|
||||
- Admin-alert coverage: only `job.failed` and `scan.failed` events exist today. `task.failed` and plugin-host crash events are not published by their subsystems yet; when they are, the admin matcher needs only a map entry.
|
||||
- Dedicated `item_added` catalog event: the content matcher keys on `catalog.item.changed` (`metadata_updated`) plus a 48-hour `media_items.created_at` recency gate, because the scanner publishes no per-item add event. An explicit ingest-time event would remove the proxy heuristic.
|
||||
- New-device login notifications: deferred from v1 (no device-fingerprint store exists); password-change notifications shipped.
|
||||
@@ -0,0 +1,173 @@
|
||||
# Core Push Notifications — Design
|
||||
|
||||
Date: 2026-06-10
|
||||
Status: approved (design); implementation plan to follow
|
||||
|
||||
## Summary
|
||||
|
||||
A server-side push subsystem that mirrors the in-app notifications created in Phase 1 out to registered devices via APNs (Apple), FCM (Android), and Web Push (browsers). Delivery is durable (a DB-backed queue with a retry worker) and presence-gated (a notification only pushes to devices whose user has no live WebSocket connection after a short grace delay). This is a core-server project: the mobile clients have no push code yet, so the deliverable is the registry, queue, worker, transports, and APIs — built transport-agnostic so APNs/FCM are drop-in alongside the immediately-testable Web Push.
|
||||
|
||||
## Goals
|
||||
|
||||
- Deliver Phase-1 notifications to devices over APNs, FCM, and Web Push.
|
||||
- Durable delivery: survive restarts, retry with backoff, prune dead tokens, observable in the admin task view.
|
||||
- Presence-gated: don't buzz a device for a notification the user is actively seeing in-app.
|
||||
- Reuse the existing per-category mute preferences; add per-device push on/off. No new per-channel matrix.
|
||||
- Encrypted provider credentials via the existing settings mechanism.
|
||||
- Transport-agnostic core: one `Transport` interface, three implementations, selected by device platform.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Mobile/web client integration (separate per-client projects; this ships the server contract).
|
||||
- A separate in-app-vs-push preference matrix (category mutes + per-device toggle only).
|
||||
- Rich push (images, action buttons) — title/body/link/category payload only in v1.
|
||||
- New-device-login / security pushes beyond what the notification categories already produce.
|
||||
|
||||
## Architecture
|
||||
|
||||
New `internal/push` package and a new `internal/presence` package, plus touches to three existing seams (`notifications.Service.Create`, the events WS handler, `cmd/silo/main.go` wiring). Five independently-testable units:
|
||||
|
||||
1. **Device/token registry** — extends `user_devices` with push columns; a registration endpoint clients call to enroll/refresh/revoke a token and toggle per-device push. Owns "which devices want push and how to reach them."
|
||||
2. **Presence registry** (`internal/presence`) — process-local `user_id → live-connection refcount`, incremented/decremented by the events WS handler on connect/disconnect. Answers "does this user have a live connection right now?" Cluster-aware via a short-TTL Redis key; degrades to per-node if Redis is absent.
|
||||
3. **Enqueuer** — called from `notifications.Service.Create()` after the row commits. Resolves push-eligible devices, writes one `push_deliveries` row per device with `not_before = now + grace`.
|
||||
4. **Delivery worker** — a `PushDeliveryTask` on the existing task framework; claims due rows, re-checks presence, dispatches via the platform's transport, records outcome, prunes dead tokens.
|
||||
5. **Transports** — a `Transport` interface (`Send(ctx, target, payload) → Result`) with APNs, FCM, and Web Push implementations selected by device platform; credentials from the encrypted settings repo.
|
||||
|
||||
Data flow: `Create()` → enqueuer writes deliveries → worker (interval) claims due & un-present → transport sends → outcome / dead-token prune. The in-app path (immediate WS publish) is untouched; push is the deferred, gated mirror.
|
||||
|
||||
### Seam reference (current code)
|
||||
|
||||
- Fan-out hook: `internal/notifications/service.go` `Create()` — after `created, err := s.store.Insert(...)` succeeds and before/alongside the `hub.PublishJSON(... "notification.created" ...)` call. Enqueue is best-effort and must not block or fail `Create()`.
|
||||
- Presence increment: `internal/api/handlers/events_ws.go` — after a successful subscribe, `release := presence.Add(claims.UserID)`, `defer release()`.
|
||||
- No presence tracking exists today (confirmed); the registry is net-new.
|
||||
- `user_devices` exists (`migrations/sql/180_user_devices.sql`, PK `user_id, profile_id, device_id`) with no token column.
|
||||
- Credential encryption: `internal/catalog/encrypted_settings_repo.go` `SensitiveSettingKeys`.
|
||||
- Background-worker precedent: `internal/taskmanager/tasks/*` (e.g. the request reconciler, notifications digest/retention).
|
||||
|
||||
## Data model
|
||||
|
||||
Extend `user_devices` (a push token belongs to a device):
|
||||
|
||||
```sql
|
||||
ALTER TABLE user_devices
|
||||
ADD COLUMN push_token text NULL,
|
||||
ADD COLUMN push_transport text NULL, -- 'apns' | 'fcm' | 'webpush'
|
||||
ADD COLUMN push_enabled boolean NOT NULL DEFAULT true,
|
||||
ADD COLUMN push_token_at timestamptz NULL,
|
||||
ADD COLUMN push_failures integer NOT NULL DEFAULT 0;
|
||||
```
|
||||
|
||||
A device is push-eligible when `push_token IS NOT NULL AND push_enabled`. For `webpush`, `push_token` holds the JSON subscription object (endpoint + p256dh + auth); APNs/FCM store the bare token string. Each transport interprets its own format.
|
||||
|
||||
Delivery queue:
|
||||
|
||||
```sql
|
||||
CREATE TABLE push_deliveries (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
notification_id bigint NOT NULL REFERENCES notifications(id) ON DELETE CASCADE,
|
||||
user_id integer NOT NULL,
|
||||
device_id text NOT NULL,
|
||||
transport text NOT NULL,
|
||||
status text NOT NULL DEFAULT 'pending', -- pending|sent|failed|skipped|dead
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
not_before timestamptz NOT NULL,
|
||||
last_error text NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX push_deliveries_claim_idx
|
||||
ON push_deliveries (not_before)
|
||||
WHERE status IN ('pending','failed');
|
||||
```
|
||||
|
||||
Worker claim: `status IN ('pending','failed') AND not_before <= now()`, ordered by `not_before`, `FOR UPDATE SKIP LOCKED` (safe across nodes/concurrent workers).
|
||||
|
||||
Outcomes:
|
||||
- **sent** — terminal success.
|
||||
- **skipped** — user was present (live WS) at claim time; the in-app delivery sufficed. Terminal.
|
||||
- **failed** — soft error (timeout, 5xx, rate-limit); bump `attempts`, push `not_before` out by the backoff schedule (1m, 5m, 30m). Once the schedule is exhausted (4 total attempts) → `dead`.
|
||||
- **dead** — hard error / unregistered token; terminal. Also clears `push_token` and sets `push_enabled=false` on the `user_devices` row so the dead device stops being enqueued.
|
||||
|
||||
Deliveries do not duplicate notification content — the worker joins `notifications` at send time so an edited/expired notification sends current content (and an expired/dismissed one can be skipped). Retention: terminal deliveries older than 7 days are deleted by a pass on the existing notifications retention task.
|
||||
|
||||
Provider config (added to `SensitiveSettingKeys`, encrypted at rest):
|
||||
`push.apns.p8_key`, `push.apns.key_id`, `push.apns.team_id`, `push.apns.bundle_id`, `push.fcm.service_account_json`, `push.webpush.vapid_public`, `push.webpush.vapid_private`, `push.webpush.subject`. A transport with missing/blank config reports itself unconfigured; its deliveries short-circuit to `skipped` (reason: unconfigured) rather than erroring in a loop.
|
||||
|
||||
## API
|
||||
|
||||
Device push registration (authenticated user routes; device identified by the existing `X-Silo-Device-Id` header + `user_id` from claims — never from the body):
|
||||
|
||||
```
|
||||
PUT /api/v1/notifications/push/device {transport, token} → 204
|
||||
enroll/refresh this device's token (upsert onto the user_devices row);
|
||||
resets push_failures, sets push_enabled=true, push_token_at=now
|
||||
DELETE /api/v1/notifications/push/device → 204
|
||||
revoke: clear token, disable push for this device
|
||||
GET /api/v1/notifications/push/devices → {devices:[{device_id,name,platform,push_enabled,registered_at}]}
|
||||
PUT /api/v1/notifications/push/devices/{device_id} {enabled} → 204
|
||||
per-device on/off toggle
|
||||
GET /api/v1/notifications/push/webpush-key → {vapid_public_key} (empty when web push unconfigured)
|
||||
```
|
||||
|
||||
Admin provider config uses the existing admin settings mechanism (encrypted `push.*` keys) — no bespoke write endpoints. One read for the admin UI:
|
||||
|
||||
```
|
||||
GET /api/v1/admin/push/status → {apns:{configured:bool}, fcm:{configured:bool}, webpush:{configured:bool}}
|
||||
```
|
||||
|
||||
Status reports booleans only; secrets are never returned.
|
||||
|
||||
## Presence registry
|
||||
|
||||
`presence.Registry` interface: `Connected(userID int) bool` and `Add(userID int) (release func())`. The events WS handler calls `release := registry.Add(claims.UserID)` after a successful subscribe and `defer release()` on disconnect.
|
||||
|
||||
- In-process implementation: `map[int]int` refcount under a mutex.
|
||||
- Cluster-aware implementation: `INCR push:presence:{userID}` in Redis with a 60s TTL refreshed on the WS heartbeat; `DECR` on release; `Connected` is key `> 0`. Redis is already used by the events hub.
|
||||
- Presence failure (Redis down) fails **open** — we push — because a missed push is worse than a redundant one.
|
||||
|
||||
The worker re-checks `Connected` at claim time; a present user's due delivery becomes `skipped`. The `not_before` grace (default 30s) means an actively-in-app user almost never gets a redundant push, while a user who closed the app moments ago does.
|
||||
|
||||
## Error handling & delivery semantics
|
||||
|
||||
- Enqueuer is best-effort; failure to write deliveries logs and returns. `Create()` is never blocked or failed by push.
|
||||
- Worker isolates per-delivery: one device's failure never blocks the batch.
|
||||
- Soft errors → `failed` + exponential backoff on `not_before` (1m, 5m, 30m), then `dead` after 4 total attempts.
|
||||
- Hard errors (`Unregistered`/`InvalidToken`/FCM `NOT_FOUND`) → immediate `dead` + clear token + disable push on `user_devices`.
|
||||
- Unconfigured transport → its deliveries short-circuit to `skipped` (reason logged once, not per-row spam).
|
||||
- Rate-limit responses park that transport briefly (honor `Retry-After`).
|
||||
- Child-profile suppression applied at enqueue: categories in {request, system, admin} are never enqueued for a child profile (same rule as the web toast suppression).
|
||||
- APNs/FCM payloads set a collapse/thread key = notification id to avoid duplicate banners across retries.
|
||||
|
||||
## Security
|
||||
|
||||
- Provider secrets live only in the encrypted settings repo; never logged, never returned by any read endpoint. `/admin/push/status` reports booleans.
|
||||
- Token registration is authenticated and device-scoped to the caller's `user_id` (from claims).
|
||||
- Push payload carries title/body/link/category + notification id — no content beyond what the notification already holds.
|
||||
|
||||
## Testing
|
||||
|
||||
- Worker tested with a fake `Transport` asserting outcome transitions: sent, skipped (present), failed-with-backoff, dead-prunes-token. No network.
|
||||
- Each real transport unit-tested against a mocked HTTP endpoint (request shape, auth header, error-code mapping) — standard repo pattern.
|
||||
- Enqueuer tested with a fake device store + presence stub: eligible-device resolution, child suppression, per-device disable, `not_before` grace.
|
||||
- Presence registry: refcount add/release, Connected transitions, Redis-absent fallback.
|
||||
- Store tests against Postgres: claim query with `SKIP LOCKED`, dead-token prune, retention.
|
||||
- One integration test: `Create() → delivery row → worker → fake transport → sent`.
|
||||
- Handler tests: registration auth-scoping, per-device toggle, `/admin/push/status` unconfigured states, `webpush-key` empty when unconfigured.
|
||||
|
||||
## Implementation sequencing
|
||||
|
||||
1. Migration (user_devices columns + push_deliveries) and domain types.
|
||||
2. `internal/presence` registry (in-process + Redis) and WS-handler wiring.
|
||||
3. Device registry store + registration/toggle endpoints.
|
||||
4. Enqueuer + hook into `Create()`.
|
||||
5. `Transport` interface + fake; delivery worker + task registration; outcome/backoff/dead-prune.
|
||||
6. Real transports (Web Push first — end-to-end testable today; then APNs, then FCM) + encrypted config keys + `/admin/push/status`.
|
||||
7. Retention pass + full verification.
|
||||
|
||||
## Future work
|
||||
|
||||
- Client integration: Apple (APNs token registration + handling), Android (FCM), web (service worker + VAPID subscription). Each binds to the frozen registration API.
|
||||
- Presence-aware in-app→push escalation tuning (per-category grace, quiet hours).
|
||||
- Rich push (images from `item_id` artwork, action buttons).
|
||||
- Per-channel preference matrix if category+device proves too coarse.
|
||||
- Shared-device push semantics: a physical device registers one OS token but one user_devices row per profile. For user-wide notifications, EligibleDevices currently returns one delivery per profile-row → the same token is sent to more than once (the per-notification collapse key dedupes the on-device banner, so the user sees one, but the sends are redundant). And child-category suppression on a device shared by a child and an adult profile is ambiguous because push cannot know the active profile at send time. Both resolve with the active-profile/claims-enrichment work also needed for WS child-filtering: carry the active profile identity so EligibleDevices can dedupe by token and apply the correct profile's suppression.
|
||||
@@ -0,0 +1,120 @@
|
||||
# Core Push — Web Client & Admin Config — Design
|
||||
|
||||
Date: 2026-06-10
|
||||
Status: approved (design); implementation plan to follow
|
||||
|
||||
## Summary
|
||||
|
||||
The core-buildable slice of Phase 3 (client work): a Web Push browser client and an admin push-provider config UI, both living in the silo-server web app plus one ~10-line Go endpoint. This makes the push subsystem operable end to end from the browser — an operator enters provider credentials in an admin page, a user enables push on their browser, and notifications arrive as OS banners with the app closed. The mobile clients (silo-apple APNs, silo-android FCM) remain out of scope; they are separate per-repo projects that consume the same frozen registration API.
|
||||
|
||||
This delivers user-facing notification delivery to browsers. It is not a plugin-communication channel; plugins relate to notifications only via the existing `notifications.send` event contract (a plugin produces a user notification, which then flows to in-app + push).
|
||||
|
||||
## Goals
|
||||
|
||||
- Web Push delivery to browsers: service worker, subscription, receive, click-through.
|
||||
- A per-device "push on this device" control and a device-management list in user settings.
|
||||
- An admin page to configure APNs / FCM / Web Push provider credentials with readiness status.
|
||||
- VAPID key generation from the admin UI.
|
||||
- Reuse everything already built: the frozen push registration API, the encrypted settings write path, the Phase-2 notifications settings page and api client.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Mobile client integration (silo-apple / silo-android) — separate projects against the frozen API.
|
||||
- A PWA framework / offline caching (vite-plugin-pwa) — a static service worker covers push; offline is out of scope.
|
||||
- Any change to the push delivery worker, transports, or registration API (all frozen).
|
||||
- Rich push (images, action buttons).
|
||||
|
||||
## Frozen dependencies (already on this branch)
|
||||
|
||||
Push registration API (`internal/api/handlers/push.go`, routes in `router.go`):
|
||||
|
||||
```
|
||||
PUT /api/v1/notifications/push/device {transport, token} → 204 (RequireProfile)
|
||||
DELETE /api/v1/notifications/push/device → 204 (RequireProfile)
|
||||
GET /api/v1/notifications/push/devices → {devices:[DeviceInfo]}
|
||||
PUT /api/v1/notifications/push/devices/{device_id} {enabled} → 204
|
||||
GET /api/v1/notifications/push/webpush-key → {vapid_public_key}
|
||||
GET /api/v1/admin/push/status → {apns,fcm,webpush: bool}
|
||||
```
|
||||
|
||||
`DeviceInfo`: `{device_id, name, platform, transport, push_enabled, registered_at}`. The api client (`web/src/api/client.ts`) auto-attaches `X-Silo-Device-Id` and `X-Profile-Id`. WebPush tokens are stored as the JSON subscription object.
|
||||
|
||||
Admin settings write path: `PUT /api/v1/admin/settings/{key}` `{value}` (`admin.go` HandleUpdateSetting) — all `push.*` keys are pre-registered sensitive in `encrypted_settings_repo.go` (encrypted at rest, value redacted from reads). No new write endpoint needed. Sensitive-status hook (`useAdminSensitiveStatus`) reports which keys are stored.
|
||||
|
||||
Web push server transport already sends payload `{id, title, body, link, category}` with `Topic: "n"+base36(notificationID)` for collapse.
|
||||
|
||||
## Architecture
|
||||
|
||||
Six units (five web, one tiny Go endpoint):
|
||||
|
||||
1. **Service worker** — `web/public/sw.js` (static). `push` → `showNotification` from payload; `notificationclick` → focus an open app tab (postMessage the link) or `openWindow(link)`; `pushsubscriptionchange` → re-subscribe with the cached VAPID key and re-register (best-effort token refresh).
|
||||
2. **SW registration** — in `web/src/main.tsx`, registers `/sw.js` when `serviceWorker` + `PushManager` are supported; no-op otherwise.
|
||||
3. **`usePushDevice` hook** — `web/src/hooks/usePushDevice.ts`. Owns the browser-push lifecycle and exposes a status state machine + `enable()`/`disable()`.
|
||||
4. **Settings UI** — extend `web/src/pages/settings/NotificationsSettings.tsx`: "Push on this device" control + a device-management list (per-device enable/disable).
|
||||
5. **Admin push-config page** — `web/src/pages/admin-settings/PushSettings.tsx`: three credential cards + VAPID keygen + readiness badges.
|
||||
6. **VAPID keygen endpoint** — `POST /api/v1/admin/push/generate-vapid-keys` (RequireAdmin), ~10 lines in `push.go`. The only net-new server code.
|
||||
|
||||
## Push client data flow
|
||||
|
||||
`usePushDevice` reports `status`:
|
||||
- **unsupported** — no serviceWorker/PushManager, or iOS Safari not installed as a PWA. UI explains; no action.
|
||||
- **blocked** — `Notification.permission === "denied"`. UI explains how to re-enable in site settings (cannot un-deny programmatically).
|
||||
- **off** — supported, not subscribed. Toggle → `enable()`.
|
||||
- **on** — subscribed and registered. Toggle → `disable()`.
|
||||
- **pending** — transition in flight; control disabled.
|
||||
|
||||
`enable()`: `Notification.requestPermission()` → if granted, `navigator.serviceWorker.ready` → `GET /notifications/push/webpush-key` → `pushManager.subscribe({ userVisibleOnly: true, applicationServerKey: urlBase64ToUint8Array(vapidPublic) })` → `PUT /notifications/push/device { transport: "webpush", token: JSON.stringify(subscription.toJSON()) }`. The api client supplies device + profile headers (satisfies the route's RequireProfile). Cache the VAPID public key (Cache Storage / IndexedDB) so the SW can re-subscribe on `pushsubscriptionchange` without the app open. Success → `on`, toast.
|
||||
|
||||
`disable()`: `pushManager.getSubscription()` → `unsubscribe()` (best-effort) → `DELETE /notifications/push/device` → `off`, toast.
|
||||
|
||||
Receiving (SW): payload `{id,title,body,link,category}` → `showNotification(title, { body, data:{link}, tag:"n"+id })`. The `tag` is an on-device display-dedup keyed on the notification id (a redelivery of the same notification replaces the visible banner rather than stacking); it is independent of the server's Web Push `Topic` header (which dedups in the push-service queue). `notificationclick` → focus matching-origin client + postMessage `{type:"notification-click", link}`, else `openWindow(link ?? "/notifications")`.
|
||||
|
||||
Idempotency / healing: `enable()` is safe to re-run (server upserts); a stale local subscription with no server row heals on re-enable. Permission-granted-but-subscribe-fails → toast, stay `off`.
|
||||
|
||||
## Admin config page
|
||||
|
||||
`PushSettings.tsx`, registered in `AdminSettingsLayout` `SETTINGS_GROUPS` (Connections group). Pattern copied from `IntegrationsSettings.tsx` (`SettingField` + `CredentialStatus` + `useUpdateServerSetting`). Cards:
|
||||
|
||||
- **Web Push** (leads — usable today): `push.webpush.vapid_public`, `push.webpush.vapid_private`, `push.webpush.subject`. "Generate keys" button → `POST /admin/push/generate-vapid-keys` → fills public/private fields (not auto-saved; admin reviews then saves each). Subject defaults to a `mailto:` hint.
|
||||
- **APNs**: `push.apns.p8_key` (textarea), `push.apns.key_id`, `push.apns.team_id`, `push.apns.bundle_id`.
|
||||
- **FCM**: `push.fcm.service_account_json` (textarea).
|
||||
|
||||
Readiness: card header badge from `GET /admin/push/status`; per-field "configured, leave blank to keep" from the sensitive-status hook (secrets never returned). Each save = `PUT /admin/settings/{key}` `{value}` via `useUpdateServerSetting`; encryption automatic; badges refetch after save.
|
||||
|
||||
VAPID keygen endpoint: generates and returns `{vapid_public, vapid_private}`; does not persist. Handler test: non-empty pair, admin-gated.
|
||||
|
||||
### Open decision — "Send test push" button
|
||||
|
||||
OPTION (admin's call at spec review): a "Send test push" button that creates a one-off system notification addressed to the admin themselves to verify config→subscribe→receive end to end from the UI. Requires one more net-new server endpoint (`POST /api/v1/admin/push/test` → `notifications.Create` a system notification for the caller). Recommended for operability (turns "did I configure this right?" into one click) but it is the only other net-new server surface. **Decision: include it** unless cut at review. (Implementation plan will gate this behind the decision.)
|
||||
|
||||
## Error handling
|
||||
|
||||
- Capability gating (unsupported, denied) are UI states, not errors; iOS Safari shows "Add to Home Screen to enable push."
|
||||
- Transient subscribe/register failures → toast, leave `off`; nothing half-committed (server PUT is source of truth, idempotent).
|
||||
- SW resilience: tolerate partial payload (generic title fallback); `notificationclick` falls back to `/notifications`; failed `pushsubscriptionchange` re-register logs and retries on next app load (browser stops receiving until reopened — never a crash).
|
||||
- Admin page: save failures toast; readiness badges always reflect server truth (refetch after save); partial config shows "Not configured" until complete; generate-keys failure leaves fields untouched.
|
||||
|
||||
## Testing
|
||||
|
||||
- `usePushDevice` — unit tests with mocked serviceWorker/PushManager/Notification/api client: status derivation, enable() happy path (permission→key→subscribe→PUT body), permission-denied short-circuit, disable() (unsubscribe + DELETE), idempotent re-enable.
|
||||
- Settings UI — component tests: each status renders the right control; device-list rows + per-device toggle mutation.
|
||||
- Service worker — vitest over the pure handlers: payload→showNotification-args mapping, click→target-URL logic. DOM wiring exercised in the manual smoke.
|
||||
- Admin page — component tests: three cards render, badges reflect `/admin/push/status`, save calls `PUT /admin/settings/{key}` with the right key, generate-keys fills fields.
|
||||
- Server — VAPID keygen handler test (non-empty pair, admin-gated); test-push endpoint handler test (if included).
|
||||
- Manual smoke (documented in plan): real browser — configure VAPID, enable push, trigger a notification, confirm OS banner with app closed, click → deep link.
|
||||
|
||||
## Implementation sequencing
|
||||
|
||||
1. Server: VAPID keygen endpoint (+ test-push endpoint if kept).
|
||||
2. Web: service worker + registration + VAPID-key caching.
|
||||
3. Web: `usePushDevice` hook + tests.
|
||||
4. Web: NotificationsSettings push control + device list.
|
||||
5. Web: admin PushSettings page + nav.
|
||||
6. Verification + manual browser smoke.
|
||||
|
||||
## Future work
|
||||
|
||||
- Mobile clients (APNs/FCM) against the frozen API.
|
||||
- Shared-device token dedup + active-profile child-suppression (recorded in the push server spec) — informs whether multiple browsers/profiles dedupe.
|
||||
- Per-category push targeting if category mutes prove too coarse.
|
||||
- PWA/offline (vite-plugin-pwa) if the app later wants installable offline behavior.
|
||||
@@ -0,0 +1,78 @@
|
||||
# On-view description translation + per-profile metadata language — design
|
||||
|
||||
**Date:** 2026-06-10
|
||||
**Status:** Approved direction (per-profile preference; auto with button fallback; any user behind an opt-in setting)
|
||||
**Builds on:** `docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md`
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
## Problem
|
||||
|
||||
Metadata translation currently triggers from the admin editor or the per-library
|
||||
refresh fallback, always targeting the library's metadata language. A viewer whose
|
||||
language differs from the library's (or whose library was never auto-translated) has
|
||||
no way to get descriptions in their language, and there is no per-person language
|
||||
preference at all — presentation language is library-scoped.
|
||||
|
||||
## Design
|
||||
|
||||
### Per-profile preferred metadata language
|
||||
|
||||
- New column `user_profiles.preferred_metadata_language text NOT NULL DEFAULT ''`
|
||||
(empty = inherit the library's metadata language). Threaded through the profile
|
||||
model/repo/API exactly like the existing `language` (UI language) field, plus a
|
||||
picker in the profile settings UI with an "Inherit library default" option.
|
||||
- `access.Scope` gains `PreferredMetadataLanguage`, populated when the access
|
||||
resolver loads the profile, so every request already carries it.
|
||||
- `catalog.AccessFilter` gains `ProfilePreferredLanguage`, and presentation language
|
||||
resolution becomes: explicit `PresentationLanguage` (request param) →
|
||||
**profile preference** → library `metadata_language`. Applied wherever the scope
|
||||
is available (native API and jellycompat), so all clients get profile-localized
|
||||
metadata server-side.
|
||||
|
||||
### "Not in your language" signal
|
||||
|
||||
`ItemDetail` gains `pending_translation_language` (omitted when empty): set when the
|
||||
resolved presentation language differs from the item's default metadata language,
|
||||
the base overview is non-empty, and no localization overview exists for that
|
||||
language. It is pure data — clients combine it with the AI feature status to decide
|
||||
what to render.
|
||||
|
||||
### On-view trigger
|
||||
|
||||
- New setting `metadata_ai.on_view`: `off` (default) | `button` | `auto`, surfaced
|
||||
on `GET /api/v1/metadata/ai/status` as `on_view`.
|
||||
- New authenticated endpoint `POST /api/v1/items/{id}/translate-description`
|
||||
`{target_language}` — any profile, gated by `metadata_ai` readiness +
|
||||
`on_view != off`, item access enforced with the same `EnsureAccessible` check the
|
||||
watched-state endpoints use. The client echoes back the language the server
|
||||
reported in `pending_translation_language`.
|
||||
- The endpoint goes through a new `Service.RequestOnView`, which adds one guard on
|
||||
top of the existing pipeline: if the latest job for (content, language) failed
|
||||
within a cooldown window, return it instead of enqueuing — a broken endpoint must
|
||||
not be retried on every page view. Existing guarantees do the rest: in-flight
|
||||
idempotency collapses concurrent viewers onto one job, skip-if-filled makes
|
||||
repeats free, and the shared semaphore caps endpoint load.
|
||||
|
||||
### Web UX
|
||||
|
||||
On the detail page, when `pending_translation_language` is set:
|
||||
|
||||
- `on_view=auto`: fire the request on view (once per item), pulse-animate the
|
||||
description while waiting, and poll the detail query until the flag clears (the
|
||||
job's first batch translates the item's own overview, so this lands in seconds
|
||||
even when a series job continues episodes in the background). Time out after ~45 s
|
||||
and show the original text — no error states in the viewer's face.
|
||||
- `on_view=button`: a small "Translate" chip near the description triggers the same
|
||||
flow on click.
|
||||
|
||||
No job-status endpoint for viewers: completion is observed as the flag clearing on
|
||||
refetch; failures surface only as the shimmer timing out (the cooldown stops
|
||||
re-triggering).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Per-profile language in the Android/Apple UIs (server-side serving means they
|
||||
already *receive* localized metadata; surfacing the trigger there is a client
|
||||
follow-up using the same endpoint).
|
||||
- Translating titles, or any admin-side behavior changes.
|
||||
@@ -0,0 +1,65 @@
|
||||
# Home Notification Bars — Design
|
||||
|
||||
Date: 2026-06-11
|
||||
Status: approved (design); implementation to follow
|
||||
|
||||
## Summary
|
||||
|
||||
Surface notifications at the top of the Home page so they're seen without opening the inbox tab, with system-level **announcements** visually distinguished from personal notifications. Two slim, dismissible bars stack at the top of Home: an **announcement bar** (distinct accent + 📢 + "Announcement" label) above a **notification strip** (the most recent unread personal notification). Both reuse existing hooks; no server changes. Web-only, additive.
|
||||
|
||||
## Goals
|
||||
|
||||
- Show the most recent unread notification high on the Home page, not only behind the bell/inbox.
|
||||
- Make admin **announcements** clearly distinct from request/content/system/admin notifications, via a separate styled bar.
|
||||
- Dismiss (✕) marks read and advances to the next unread; bars hide when nothing's unread.
|
||||
- Reuse the Phase-2 notifications hooks; inherit per-profile + child filtering and live WebSocket updates for free.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No server/API changes (the data already distinguishes `category: "announcement"`).
|
||||
- No change to the bell, inbox page, or toast (carrying the 📢 styling there is a possible follow-up, out of scope here).
|
||||
- Not app-wide — Home page only.
|
||||
|
||||
## Components
|
||||
|
||||
Two components in `web/src/components/home/` (or alongside Home), each small and independently testable, plus one mount edit.
|
||||
|
||||
### AnnouncementBar (`web/src/components/AnnouncementBar.tsx`)
|
||||
- Data: `useNotificationsList({ unread: true, category: "announcement" })` → the first (newest) item. Reliably finds the latest unread announcement regardless of how many other notifications are unread.
|
||||
- Render: a distinct bar using the app's accent tokens (reads as a system notice, not a personal row) — a 📢 icon, an "ANNOUNCEMENT" label, the title, truncated body, relative time, and a ✕.
|
||||
- Empty: renders nothing when there's no unread announcement.
|
||||
- ✕: `useMarkRead().mutate({ ids: [id] })` → the unread query updates → the next unread announcement shows, or the bar hides. Marking read also drops the bell badge (shared cache).
|
||||
- Click (anywhere but ✕): navigate to the notification's `link` (or `/notifications` when it has none) and mark it read.
|
||||
|
||||
### NotificationStrip (`web/src/components/NotificationStrip.tsx`)
|
||||
- Data: `useNotificationsList({ unread: true })` → the first item whose `category !== "announcement"` (so announcements live only in the bar above, never duplicated here).
|
||||
- Render: the slim strip — 🔔 icon, title, truncated body, relative time, ✕. Lighter styling than the announcement bar.
|
||||
- Empty: renders nothing when there's no unread non-announcement.
|
||||
- ✕ / click: same behavior as the bar (mark read + advance / navigate + mark read).
|
||||
|
||||
### Mount (`web/src/pages/Home.tsx`)
|
||||
At the top of the Home content region (near the existing `heroSlot` / `TasteSeedBanner`), render `<AnnouncementBar />` then `<NotificationStrip />` as the first elements, so they sit above the content rows. When both are empty, Home renders exactly as today.
|
||||
|
||||
## Data flow & shared behavior
|
||||
|
||||
- Both bars read through TanStack Query; the announcement bar's query (`{unread:true, category:"announcement"}`) and the strip's (`{unread:true}`) are distinct cache keys — two small fetches, both already used elsewhere.
|
||||
- Live updates: `useNotificationsLive` already invalidates the notifications list on `notification.created`, so both bars reactively show the newest unread without a reload.
|
||||
- Filtering: the list endpoint already scopes to the active profile and suppresses child-restricted categories server-side, so the bars inherit correct per-profile behavior.
|
||||
- The relative-time helper is the existing `timeAgo` (with the absolute-date fallback for old timestamps).
|
||||
|
||||
## Error handling
|
||||
|
||||
- While the query is loading, render nothing (no skeleton flash for a slim bar).
|
||||
- A mark-read failure surfaces the existing mutation `onError` toast; the bar stays until the next successful read.
|
||||
- A notification with no `link` navigates to `/notifications` rather than a dead link.
|
||||
|
||||
## Testing
|
||||
|
||||
- `AnnouncementBar.test.tsx`: renders the latest unread announcement with the 📢/label styling; ✕ calls `markRead({ids:[id]})`; clicking navigates (mocked `useNavigate`) to the link and marks read; renders nothing when no unread announcement.
|
||||
- `NotificationStrip.test.tsx`: shows the newest unread non-announcement (skips an announcement at the head of the list); ✕ marks read; click navigates + marks read; renders nothing when no unread non-announcement.
|
||||
- Existing suites stay green; tsc + lint clean.
|
||||
|
||||
## Future work
|
||||
|
||||
- Carry the 📢 announcement styling into the bell dropdown and inbox rows for a consistent identity.
|
||||
- Optional: a per-user "hide the home bars" preference if they prove noisy.
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/Silo-Server/silo-server
|
||||
|
||||
go 1.26.3
|
||||
go 1.26.4
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
@@ -14,12 +14,13 @@ require (
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
golang.org/x/crypto v0.50.0
|
||||
golang.org/x/sys v0.43.0
|
||||
golang.org/x/crypto v0.52.0
|
||||
golang.org/x/sys v0.45.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0
|
||||
github.com/abadojack/whatlanggo v1.0.1
|
||||
github.com/go-chi/cors v1.2.2
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
@@ -29,32 +30,45 @@ require (
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
github.com/pressly/goose/v3 v3.27.1
|
||||
github.com/sideshow/apns2 v0.25.0
|
||||
github.com/zishang520/socket.io/v2 v2.5.0
|
||||
go.n16f.net/thumbhash v1.1.0
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/image v0.41.0
|
||||
golang.org/x/net v0.55.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dunglas/httpsfv v1.1.0 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/oklog/run v1.1.0 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.53.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.60.0 // indirect
|
||||
github.com/quic-go/webtransport-go v0.10.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||
@@ -62,16 +76,11 @@ require (
|
||||
github.com/zishang520/engine.io/v2 v2.5.0 // indirect
|
||||
github.com/zishang520/socket.io-go-parser/v2 v2.5.0 // indirect
|
||||
github.com/zishang520/webtransport-go v0.9.1 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.0
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.6.0
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
@@ -90,15 +99,16 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/sony/sonyflake v1.3.0
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/text v0.36.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/grpc v1.79.3
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/text v0.37.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/grpc v1.81.1
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
replace github.com/zishang520/webtransport-go => ./internal/compat/zishang520-webtransport-go
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.0 h1:DJkRROQfr/kfwnF5dUdkhmbCso1KDLZc7uK7YfJnaO0=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s=
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.6.0 h1:Gi9TdH9kt7b8X4xRXH493/nSYb9n0GO4VCWmlll0hKI=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.6.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
@@ -45,16 +51,19 @@ github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54=
|
||||
github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||
@@ -67,10 +76,15 @@ github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0=
|
||||
github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA=
|
||||
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
|
||||
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -108,8 +122,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
@@ -125,10 +139,12 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
@@ -140,6 +156,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
|
||||
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
@@ -147,32 +165,46 @@ github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNs
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc=
|
||||
github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.27.1 h1:6uEvcprBybDmW4hcz3gYujhARhye+GoWKhEWyzD5sh4=
|
||||
github.com/pressly/goose/v3 v3.27.1/go.mod h1:maruOxsPnIG2yHHyo8UqKWXYKFcH7Q76csUV7+7KYoM=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.53.0 h1:QHX46sISpG2S03dPeZBgVIZp8dGagIaiu2FiVYvpCZI=
|
||||
github.com/quic-go/quic-go v0.53.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
|
||||
github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0=
|
||||
github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk=
|
||||
github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI=
|
||||
github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/sideshow/apns2 v0.25.0 h1:XOzanncO9MQxkb03T/2uU2KcdVjYiIf0TMLzec0FTW4=
|
||||
github.com/sideshow/apns2 v0.25.0/go.mod h1:7Fceu+sL0XscxrfLSkAoH6UtvKefq3Kq1n4W3ayQZqE=
|
||||
github.com/sony/sonyflake v1.3.0 h1:tiB4Dlp0lnmKp/h6BLXA14P8Qi+LYS9+0QRpcrKHvg4=
|
||||
github.com/sony/sonyflake v1.3.0/go.mod h1:LORtCywH/cq10ZbyfhKrHYgAUGH7mOBa76enV9txy/Y=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
@@ -199,6 +231,7 @@ github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHg
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
github.com/zishang520/engine.io-go-parser v1.3.2 h1:aEVrhQVhfk99Ct6htNffgHydUBC4dGclO/OXPz5CSy0=
|
||||
@@ -209,73 +242,134 @@ github.com/zishang520/socket.io-go-parser/v2 v2.5.0 h1:uGKTwcH2qrrs9uwzfCo3ems+q
|
||||
github.com/zishang520/socket.io-go-parser/v2 v2.5.0/go.mod h1:GK9GIIs/KQbBKfnxgZJMBYSlsFemB2rL7EFUn0kmTfM=
|
||||
github.com/zishang520/socket.io/v2 v2.5.0 h1:+KdZLbl4wWVzyI84RG+h21t8OY4ifz/Oo6qwgt9zHl8=
|
||||
github.com/zishang520/socket.io/v2 v2.5.0/go.mod h1:+GyoPyakXDS6KsW81RAQpDA9+mJBXbcYcQ+Itx2D+rU=
|
||||
github.com/zishang520/webtransport-go v0.9.1 h1:Y3gqPM8cIDvQILsTyXJ5G9fp2PYqGqLI2z+QXpgboQc=
|
||||
github.com/zishang520/webtransport-go v0.9.1/go.mod h1:IgNAD6qLe3oWu7MSSkjusRNftpvjYxWjI4LmoH4VEyY=
|
||||
go.n16f.net/thumbhash v1.1.0 h1:aBEvuAd4yiwzeQ7Sm4BZoHJYbrQ1ewjrmrRlCE79snk=
|
||||
go.n16f.net/thumbhash v1.1.0/go.mod h1:mo9pP7WtfdV9ojIamGFR/Vc0PaPA2l0CUtmYQf/SweU=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
|
||||
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
|
||||
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
|
||||
golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ=
|
||||
google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -285,3 +379,11 @@ gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
|
||||
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
|
||||
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
|
||||
modernc.org/libc v1.72.1 h1:db1xwJ6u1kE3KHTFTTbe2GCrczHPKzlURP0aDC4NGD0=
|
||||
modernc.org/libc v1.72.1/go.mod h1:HRMiC/PhPGLIPM7GzAFCbI+oSgE3dhZ8FWftmRrHVlY=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
|
||||
@@ -73,6 +73,7 @@ func (r *Resolver) Resolve(ctx context.Context, input ResolveInput) (Scope, erro
|
||||
|
||||
scope.MaxContentRating = profile.MaxContentRating
|
||||
scope.MaxPlaybackQuality = MinQuality(scope.MaxPlaybackQuality, NormalizePlaybackQuality(profile.MaxPlaybackQuality))
|
||||
scope.PreferredMetadataLanguage = profile.PreferredMetadataLanguage
|
||||
scope.AllowedLibraryIDs, scope.LibrariesRestricted = effectiveLibraries(user.LibraryIDs, profile)
|
||||
scope.ProfileVerified = profile.PINHash == "" || input.SkipPINVerification
|
||||
|
||||
|
||||
@@ -9,8 +9,11 @@ type Scope struct {
|
||||
LibrariesRestricted bool
|
||||
MaxContentRating string
|
||||
MaxPlaybackQuality string
|
||||
PolicyRevision int64
|
||||
ProfileVerified bool
|
||||
// PreferredMetadataLanguage is the profile's metadata (presentation)
|
||||
// language; "" inherits the library's metadata language.
|
||||
PreferredMetadataLanguage string
|
||||
PolicyRevision int64
|
||||
ProfileVerified bool
|
||||
}
|
||||
|
||||
// ResolveInput is the request input for resolving a viewer access scope.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package adminjob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// CancelRegistry bridges HTTP cancellation requests to in-process job contexts.
|
||||
// Queued jobs are cancelled in the repository; running jobs need their active
|
||||
// context cancelled so executors stop doing work before the row is finalized.
|
||||
type CancelRegistry struct {
|
||||
mu sync.Mutex
|
||||
cancels map[string]cancelEntry
|
||||
}
|
||||
|
||||
type cancelEntry struct {
|
||||
cancel context.CancelFunc
|
||||
token *struct{}
|
||||
}
|
||||
|
||||
func NewCancelRegistry() *CancelRegistry {
|
||||
return &CancelRegistry{cancels: make(map[string]cancelEntry)}
|
||||
}
|
||||
|
||||
func (r *CancelRegistry) Register(jobID string, cancel context.CancelFunc) func() {
|
||||
if r == nil || jobID == "" || cancel == nil {
|
||||
return func() {}
|
||||
}
|
||||
token := &struct{}{}
|
||||
r.mu.Lock()
|
||||
r.cancels[jobID] = cancelEntry{cancel: cancel, token: token}
|
||||
r.mu.Unlock()
|
||||
|
||||
return func() {
|
||||
r.mu.Lock()
|
||||
if entry := r.cancels[jobID]; entry.token == token {
|
||||
delete(r.cancels, jobID)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *CancelRegistry) Cancel(jobID string) bool {
|
||||
if r == nil || jobID == "" {
|
||||
return false
|
||||
}
|
||||
r.mu.Lock()
|
||||
cancel := r.cancels[jobID].cancel
|
||||
r.mu.Unlock()
|
||||
if cancel == nil {
|
||||
return false
|
||||
}
|
||||
cancel()
|
||||
return true
|
||||
}
|
||||
@@ -542,17 +542,19 @@ func (e *ItemRefreshExecutor) Execute(ctx context.Context, req ItemRefreshReques
|
||||
e.publish(cache.EventMetadataUpdated, refreshContentID)
|
||||
if e.realtimeHub != nil {
|
||||
if scanResult != nil && (scanResult.New > 0 || scanResult.Updated > 0 || scanResult.Missing > 0 || matched > 0) {
|
||||
_ = e.realtimeHub.PublishLibraryChanged(ctx, notifications.LibraryChangeEvent{
|
||||
_ = e.realtimeHub.PublishCatalogLibraryChanged(ctx, notifications.LibraryChangeEvent{
|
||||
LibraryID: req.ScanFolderID,
|
||||
Reason: "item_refresh",
|
||||
New: scanResult.New,
|
||||
Updated: scanResult.Updated,
|
||||
Missing: scanResult.Missing,
|
||||
MatchedFiles: matched,
|
||||
})
|
||||
}
|
||||
_ = e.realtimeHub.PublishMetadataUpdated(ctx, notifications.MetadataUpdateEvent{
|
||||
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
|
||||
LibraryID: req.ScanFolderID,
|
||||
ContentID: refreshContentID,
|
||||
Change: "metadata_updated",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -251,8 +251,9 @@ func (e *LibraryRefreshExecutor) Execute(
|
||||
if result.RefreshedOK > 0 || result.PipelineOK > 0 {
|
||||
e.publish(cache.EventMetadataUpdated, strconv.Itoa(req.LibraryID))
|
||||
if e.realtimeHub != nil {
|
||||
_ = e.realtimeHub.PublishMetadataUpdated(ctx, notifications.MetadataUpdateEvent{
|
||||
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
|
||||
LibraryID: req.LibraryID,
|
||||
Change: "metadata_updated",
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -321,6 +322,7 @@ func (e *LibraryRefreshExecutor) refreshItemsWithIDs(
|
||||
result.RefreshedFailed++
|
||||
} else {
|
||||
result.RefreshedOK++
|
||||
e.publishCatalogItemChanged(ctx, result.LibraryID, itemResult.contentID)
|
||||
}
|
||||
advance("Refreshing items with external IDs")
|
||||
}
|
||||
@@ -386,6 +388,7 @@ func (e *LibraryRefreshExecutor) refreshUnmatchedItem(ctx context.Context, libra
|
||||
if err := e.refresher.RefreshItemForLibrary(ctx, req.RefreshContentID, req.ScanFolderID); err != nil {
|
||||
return fmt.Errorf("refresh metadata: %w", err)
|
||||
}
|
||||
e.publishCatalogItemChanged(ctx, libraryID, req.RefreshContentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -399,6 +402,17 @@ func (e *LibraryRefreshExecutor) publish(eventType, payload string) {
|
||||
})
|
||||
}
|
||||
|
||||
func (e *LibraryRefreshExecutor) publishCatalogItemChanged(ctx context.Context, libraryID int, contentID string) {
|
||||
if e == nil || e.realtimeHub == nil || contentID == "" {
|
||||
return
|
||||
}
|
||||
_ = e.realtimeHub.PublishCatalogItemChanged(ctx, notifications.MetadataUpdateEvent{
|
||||
LibraryID: libraryID,
|
||||
ContentID: contentID,
|
||||
Change: "metadata_updated",
|
||||
})
|
||||
}
|
||||
|
||||
func decodeLibraryRefreshRequest(data json.RawMessage) (LibraryRefreshRequest, error) {
|
||||
var req LibraryRefreshRequest
|
||||
if len(data) == 0 {
|
||||
|
||||
@@ -24,11 +24,13 @@ const (
|
||||
StatusRunning = "running"
|
||||
StatusCompleted = "completed"
|
||||
StatusFailed = "failed"
|
||||
StatusCancelled = "cancelled"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrJobNotFound = errors.New("admin job not found")
|
||||
ErrActiveJobConflict = errors.New("admin job already active for type")
|
||||
ErrJobNotCancellable = errors.New("admin job is not cancellable")
|
||||
)
|
||||
|
||||
type ActiveJobConflictError struct {
|
||||
@@ -471,6 +473,83 @@ func (r *Repository) Fail(ctx context.Context, id string, input FailJobInput) er
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Repository) Cancel(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error) {
|
||||
if message == "" {
|
||||
message = "Admin job cancelled"
|
||||
}
|
||||
job, err := scanAdminJob(r.pool.QueryRow(ctx, `
|
||||
UPDATE admin_jobs
|
||||
SET status = $2,
|
||||
message = $3,
|
||||
error_message = '',
|
||||
completed_at = NOW(),
|
||||
heartbeat_at = NOW(),
|
||||
expires_at = $4,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
AND status IN ($5, $6)
|
||||
RETURNING `+adminJobColumns,
|
||||
id,
|
||||
StatusCancelled,
|
||||
message,
|
||||
expiresAt,
|
||||
StatusQueued,
|
||||
StatusRunning,
|
||||
))
|
||||
if err == nil {
|
||||
return job, nil
|
||||
}
|
||||
if !errors.Is(err, ErrJobNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
existing, lookupErr := r.GetByID(ctx, id)
|
||||
if lookupErr != nil {
|
||||
return nil, lookupErr
|
||||
}
|
||||
if existing.Status != StatusQueued && existing.Status != StatusRunning {
|
||||
return nil, ErrJobNotCancellable
|
||||
}
|
||||
return nil, ErrJobNotFound
|
||||
}
|
||||
|
||||
func (r *Repository) CancelQueued(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error) {
|
||||
if message == "" {
|
||||
message = "Admin job cancelled"
|
||||
}
|
||||
job, err := scanAdminJob(r.pool.QueryRow(ctx, `
|
||||
UPDATE admin_jobs
|
||||
SET status = $2,
|
||||
message = $3,
|
||||
error_message = '',
|
||||
completed_at = NOW(),
|
||||
heartbeat_at = NOW(),
|
||||
expires_at = $4,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
AND status = $5
|
||||
RETURNING `+adminJobColumns,
|
||||
id,
|
||||
StatusCancelled,
|
||||
message,
|
||||
expiresAt,
|
||||
StatusQueued,
|
||||
))
|
||||
if err == nil {
|
||||
return job, nil
|
||||
}
|
||||
if !errors.Is(err, ErrJobNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
existing, lookupErr := r.GetByID(ctx, id)
|
||||
if lookupErr != nil {
|
||||
return nil, lookupErr
|
||||
}
|
||||
if existing.Status != StatusQueued {
|
||||
return nil, ErrJobNotCancellable
|
||||
}
|
||||
return nil, ErrJobNotFound
|
||||
}
|
||||
|
||||
func (r *Repository) RequeueStaleRunning(ctx context.Context, before time.Time) (int, error) {
|
||||
tag, err := r.pool.Exec(ctx, `
|
||||
UPDATE admin_jobs
|
||||
|
||||
@@ -57,6 +57,7 @@ type Runner struct {
|
||||
heartbeatInterval time.Duration
|
||||
staleAfter time.Duration
|
||||
retention time.Duration
|
||||
cancelRegistry *CancelRegistry
|
||||
stop chan struct{}
|
||||
stopOnce sync.Once
|
||||
}
|
||||
@@ -99,10 +100,17 @@ func NewRunner(
|
||||
heartbeatInterval: 10 * time.Second,
|
||||
staleAfter: 2 * time.Minute,
|
||||
retention: 7 * 24 * time.Hour,
|
||||
cancelRegistry: NewCancelRegistry(),
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) SetCancelRegistry(registry *CancelRegistry) {
|
||||
if registry != nil {
|
||||
r.cancelRegistry = registry
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) Start() {
|
||||
go func() {
|
||||
r.requeueStaleJobs()
|
||||
@@ -329,6 +337,8 @@ func (r *Runner) executeLibraryRefresh(job *models.AdminJob) {
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), libraryRefreshTimeout)
|
||||
defer cancel()
|
||||
unregisterCancel := r.cancelRegistry.Register(job.ID, cancel)
|
||||
defer unregisterCancel()
|
||||
|
||||
heartbeatStop := make(chan struct{})
|
||||
go r.heartbeatLoop(ctx, job.ID, heartbeatStop)
|
||||
@@ -357,6 +367,10 @@ func (r *Runner) executeLibraryRefresh(job *models.AdminJob) {
|
||||
r.publishJobByID(ctx, notifications.TypeJobProgress, job.ID)
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
r.cancelJob(job.ID, current, total, "Library metadata refresh cancelled")
|
||||
return
|
||||
}
|
||||
msg := err.Error()
|
||||
if ctx.Err() != nil {
|
||||
msg = fmt.Sprintf("timed out after %s: %s", libraryRefreshTimeout, msg)
|
||||
@@ -846,3 +860,21 @@ func (r *Runner) failJob(id string, current, total int, message, errorMessage st
|
||||
}
|
||||
r.publishJobByID(ctx, notifications.TypeJobFailed, id)
|
||||
}
|
||||
|
||||
func (r *Runner) cancelJob(id string, current, total int, message string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := r.repo.UpdateProgress(ctx, id, current, total, message); err != nil {
|
||||
slog.Warn("admin jobs: failed to update cancellation progress", "job_id", id, "error", err)
|
||||
}
|
||||
job, err := r.repo.Cancel(ctx, id, message, time.Now().UTC().Add(r.retention))
|
||||
if err != nil {
|
||||
slog.Warn("admin jobs: failed to mark job cancelled", "job_id", id, "error", err)
|
||||
return
|
||||
}
|
||||
if r.realtimeHub != nil {
|
||||
if err := r.realtimeHub.PublishJob(ctx, notifications.TypeJobCancelled, job); err != nil {
|
||||
slog.Warn("admin jobs: failed to publish job cancellation", "job_id", id, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
// Package jobrunner owns the lifecycle mechanics shared by Silo's AI job
|
||||
// services (subtitle translation/ASR, metadata translation): bounded dispatch
|
||||
// off a semaphore shared across services, a heartbeat loop that keeps a row
|
||||
// alive even while the job is queued, a stale-job reaper for rows orphaned by
|
||||
// a crashed worker, and a per-job cancel registry. The services keep their own
|
||||
// job tables, validation, and run logic; this package keeps them honest about
|
||||
// concurrency and crash recovery without copy-pasting the trickiest code.
|
||||
package jobrunner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// A running job refreshes its heartbeat every HeartbeatInterval; one whose
|
||||
// heartbeat has not advanced for StaleJobThreshold is treated as orphaned
|
||||
// by a crashed worker and reaped. The margin over HeartbeatInterval avoids
|
||||
// reaping a job that is merely mid–LLM-call.
|
||||
HeartbeatInterval = 30 * time.Second
|
||||
StaleJobThreshold = 2 * time.Minute
|
||||
// How often the background reaper scans for orphaned jobs.
|
||||
ReaperInterval = time.Minute
|
||||
)
|
||||
|
||||
// Store is the minimal persistence surface the runner needs. Both AI job
|
||||
// repositories satisfy it.
|
||||
type Store interface {
|
||||
Heartbeat(ctx context.Context, id int64) error
|
||||
// ResetStaleJobs marks pending/running jobs whose heartbeat predates
|
||||
// `before` as failed with the given message. Returns rows reset.
|
||||
ResetStaleJobs(ctx context.Context, before time.Time, message string) (int64, error)
|
||||
}
|
||||
|
||||
// NewSemaphore builds the dispatch semaphore shared across runners, so the
|
||||
// configured endpoint sees one global bound regardless of job mix. size <= 0
|
||||
// falls back to 2.
|
||||
func NewSemaphore(size int) chan struct{} {
|
||||
if size <= 0 {
|
||||
size = 2
|
||||
}
|
||||
return make(chan struct{}, size)
|
||||
}
|
||||
|
||||
// Runner executes jobs with bounded concurrency, heartbeats, cancellation,
|
||||
// and crash recovery. One Runner per job table; the semaphore may be shared
|
||||
// across Runners.
|
||||
type Runner struct {
|
||||
// baseCtx is the application context; dispatched jobs and the reaper
|
||||
// derive from it so they stop when the server shuts down.
|
||||
baseCtx context.Context
|
||||
sem chan struct{}
|
||||
store Store
|
||||
// label names the job family in log lines ("subtitle ai", "metadata translation").
|
||||
label string
|
||||
logger *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
cancels map[int64]context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// New wires a runner. A nil appCtx falls back to context.Background(); a nil
|
||||
// sem gets a private default-size semaphore (used by tests; production passes
|
||||
// the shared one).
|
||||
func New(appCtx context.Context, sem chan struct{}, store Store, label string, logger *slog.Logger) *Runner {
|
||||
if appCtx == nil {
|
||||
appCtx = context.Background()
|
||||
}
|
||||
if sem == nil {
|
||||
sem = NewSemaphore(0)
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Runner{
|
||||
baseCtx: appCtx,
|
||||
sem: sem,
|
||||
store: store,
|
||||
label: label,
|
||||
logger: logger,
|
||||
cancels: make(map[int64]context.CancelFunc),
|
||||
}
|
||||
}
|
||||
|
||||
// Recover clears jobs orphaned by a crashed worker and starts a background
|
||||
// reaper that keeps doing so. Reaping is heartbeat-based (not "every active
|
||||
// job"), so it is safe when multiple instances share one database: a job
|
||||
// still being heartbeat-updated by a live worker is never reset. Call once at
|
||||
// startup; jobs and the reaper derive from the application context, so they
|
||||
// stop on shutdown.
|
||||
func (r *Runner) Recover() {
|
||||
r.reapStaleJobs()
|
||||
go r.reaperLoop()
|
||||
}
|
||||
|
||||
func (r *Runner) reaperLoop() {
|
||||
ticker := time.NewTicker(ReaperInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-r.baseCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.reapStaleJobs()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) reapStaleJobs() {
|
||||
before := time.Now().Add(-StaleJobThreshold)
|
||||
n, err := r.store.ResetStaleJobs(context.WithoutCancel(r.baseCtx), before, "interrupted by server restart")
|
||||
if err != nil {
|
||||
r.logger.Warn("failed to reset stale jobs", "jobs", r.label, "error", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
r.logger.Info("reset stale jobs", "jobs", r.label, "count", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Dispatch launches a bounded background goroutine for job id. run executes
|
||||
// once a semaphore slot is acquired; onAbort runs instead if the job is
|
||||
// cancelled (user cancel or shutdown) while still waiting for a slot. Both
|
||||
// receive a context derived from the application context that is cancelled by
|
||||
// Cancel(id) or server shutdown.
|
||||
func (r *Runner) Dispatch(id int64, run func(ctx context.Context), onAbort func(ctx context.Context)) {
|
||||
runCtx, cancel := context.WithCancel(r.baseCtx)
|
||||
r.mu.Lock()
|
||||
r.cancels[id] = cancel
|
||||
r.mu.Unlock()
|
||||
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
delete(r.cancels, id)
|
||||
r.mu.Unlock()
|
||||
cancel()
|
||||
}()
|
||||
|
||||
// Heartbeat for the whole lifetime — crucially including while queued
|
||||
// behind the semaphore — so the stale-job reaper never reaps a job that
|
||||
// is alive but merely waiting for a slot (which would otherwise mark it
|
||||
// failed and let it resurrect itself on acquire, or admit a duplicate).
|
||||
stopHeartbeat := make(chan struct{})
|
||||
defer close(stopHeartbeat)
|
||||
go r.heartbeatLoop(runCtx, id, stopHeartbeat)
|
||||
|
||||
select {
|
||||
case r.sem <- struct{}{}:
|
||||
case <-runCtx.Done():
|
||||
if onAbort != nil {
|
||||
onAbort(context.WithoutCancel(runCtx))
|
||||
}
|
||||
return
|
||||
}
|
||||
defer func() { <-r.sem }()
|
||||
|
||||
run(runCtx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Cancel cancels the in-flight goroutine for id, returning false when none is
|
||||
// registered (job on another node, or never dispatched) so the caller can
|
||||
// fall back to a best-effort terminal transition in the database.
|
||||
func (r *Runner) Cancel(id int64) bool {
|
||||
r.mu.Lock()
|
||||
cancel := r.cancels[id]
|
||||
r.mu.Unlock()
|
||||
if cancel == nil {
|
||||
return false
|
||||
}
|
||||
cancel()
|
||||
return true
|
||||
}
|
||||
|
||||
// heartbeatLoop keeps a job's heartbeat_at fresh until the job ends or the
|
||||
// context is cancelled, so the stale-job reaper only ever reaps jobs orphaned
|
||||
// by a crashed worker.
|
||||
func (r *Runner) heartbeatLoop(ctx context.Context, jobID int64, stop <-chan struct{}) {
|
||||
ticker := time.NewTicker(HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
_ = r.store.Heartbeat(context.WithoutCancel(ctx), jobID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package jobrunner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
heartbeats int
|
||||
resets int
|
||||
lastBefore time.Time
|
||||
}
|
||||
|
||||
func (s *fakeStore) Heartbeat(context.Context, int64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.heartbeats++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeStore) ResetStaleJobs(_ context.Context, before time.Time, _ string) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.resets++
|
||||
s.lastBefore = before
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s *fakeStore) snapshot() (int, time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.resets, s.lastBefore
|
||||
}
|
||||
|
||||
func TestRecoverReapsImmediatelyWithStaleCutoff(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
approxNow := time.Now()
|
||||
New(ctx, nil, store, "test", nil).Recover()
|
||||
|
||||
resets, before := store.snapshot()
|
||||
if resets < 1 {
|
||||
t.Fatalf("Recover did not reap immediately: resets=%d", resets)
|
||||
}
|
||||
want := approxNow.Add(-StaleJobThreshold)
|
||||
if diff := before.Sub(want); diff > 2*time.Second || diff < -2*time.Second {
|
||||
t.Errorf("stale cutoff = %v, want ~%v", before, want)
|
||||
}
|
||||
}
|
||||
|
||||
// One shared semaphore bounds jobs across BOTH runners: with size 1, the
|
||||
// second runner's job must not start until the first runner's job finishes.
|
||||
func TestSharedSemaphoreBoundsAcrossRunners(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sem := NewSemaphore(1)
|
||||
r1 := New(ctx, sem, &fakeStore{}, "one", nil)
|
||||
r2 := New(ctx, sem, &fakeStore{}, "two", nil)
|
||||
|
||||
release := make(chan struct{})
|
||||
aStarted := make(chan struct{})
|
||||
var bStarted atomic.Bool
|
||||
bDone := make(chan struct{})
|
||||
|
||||
r1.Dispatch(1, func(context.Context) {
|
||||
close(aStarted)
|
||||
<-release
|
||||
}, nil)
|
||||
<-aStarted
|
||||
|
||||
r2.Dispatch(2, func(context.Context) {
|
||||
bStarted.Store(true)
|
||||
close(bDone)
|
||||
}, nil)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if bStarted.Load() {
|
||||
t.Fatal("job B ran while job A held the only slot")
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-bDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("job B never ran after the slot freed")
|
||||
}
|
||||
}
|
||||
|
||||
// Cancelling a job that is queued behind the semaphore aborts it without
|
||||
// running it, via the onAbort callback.
|
||||
func TestCancelWhileQueuedCallsOnAbort(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sem := NewSemaphore(1)
|
||||
r := New(ctx, sem, &fakeStore{}, "test", nil)
|
||||
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
started := make(chan struct{})
|
||||
r.Dispatch(1, func(context.Context) {
|
||||
close(started)
|
||||
<-release
|
||||
}, nil)
|
||||
<-started
|
||||
|
||||
var ran atomic.Bool
|
||||
aborted := make(chan struct{})
|
||||
r.Dispatch(2, func(context.Context) {
|
||||
ran.Store(true)
|
||||
}, func(context.Context) {
|
||||
close(aborted)
|
||||
})
|
||||
|
||||
if !r.Cancel(2) {
|
||||
t.Fatal("Cancel(2) found no in-flight goroutine")
|
||||
}
|
||||
select {
|
||||
case <-aborted:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("onAbort never ran")
|
||||
}
|
||||
if ran.Load() {
|
||||
t.Fatal("cancelled queued job still ran")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelUnknownJobReturnsFalse(t *testing.T) {
|
||||
r := New(context.Background(), nil, &fakeStore{}, "test", nil)
|
||||
if r.Cancel(99) {
|
||||
t.Fatal("Cancel(99) = true for unknown job")
|
||||
}
|
||||
}
|
||||
|
||||
// The run context is cancelled by Cancel(id) so an in-flight job can stop.
|
||||
func TestCancelRunningJobCancelsContext(t *testing.T) {
|
||||
r := New(context.Background(), NewSemaphore(1), &fakeStore{}, "test", nil)
|
||||
started := make(chan struct{})
|
||||
stopped := make(chan struct{})
|
||||
r.Dispatch(7, func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
close(stopped)
|
||||
}, nil)
|
||||
<-started
|
||||
if !r.Cancel(7) {
|
||||
t.Fatal("Cancel(7) found no in-flight goroutine")
|
||||
}
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("running job did not observe cancellation")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package jobrunner
|
||||
|
||||
// Status is the lifecycle state shared by every AI job table.
|
||||
type Status string
|
||||
|
||||
const (
|
||||
StatusPending Status = "pending"
|
||||
StatusRunning Status = "running"
|
||||
StatusCompleted Status = "completed"
|
||||
StatusFailed Status = "failed"
|
||||
StatusCancelled Status = "cancelled"
|
||||
)
|
||||
|
||||
// Terminal reports whether a status is final.
|
||||
func (s Status) Terminal() bool {
|
||||
switch s {
|
||||
case StatusCompleted, StatusFailed, StatusCancelled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
// Package llm provides the shared OpenAI-compatible API client used by every
|
||||
// AI feature in Silo (subtitle translation, metadata translation, Whisper ASR).
|
||||
// One endpoint configuration, one retry/backoff implementation; the operator
|
||||
// can point it at OpenAI, Groq, a local Ollama/llama.cpp/faster-whisper
|
||||
// server, etc.
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config holds the connection settings for the shared OpenAI-compatible
|
||||
// endpoints. The ASR fields are optional overrides for operators who run a
|
||||
// separate Whisper-compatible server next to their chat endpoint; when empty,
|
||||
// transcription uses the chat endpoint's base URL and key.
|
||||
type Config struct {
|
||||
BaseURL string // e.g. "https://api.openai.com" (no trailing /v1)
|
||||
APIKey string // empty for keyless local servers
|
||||
ChatModel string // chat-completions model used for translation
|
||||
|
||||
ASRBaseURL string // optional; empty = BaseURL
|
||||
ASRAPIKey string // optional; empty = APIKey
|
||||
ASRModel string // audio-transcription model, e.g. "whisper-1"
|
||||
}
|
||||
|
||||
func (c Config) asrBaseURL() string {
|
||||
if c.ASRBaseURL != "" {
|
||||
return c.ASRBaseURL
|
||||
}
|
||||
return c.BaseURL
|
||||
}
|
||||
|
||||
func (c Config) asrAPIKey() string {
|
||||
if c.ASRAPIKey != "" {
|
||||
return c.ASRAPIKey
|
||||
}
|
||||
return c.APIKey
|
||||
}
|
||||
|
||||
// ChatConfigured reports whether chat completions are minimally configured.
|
||||
func (c Config) ChatConfigured() bool { return c.BaseURL != "" && c.ChatModel != "" }
|
||||
|
||||
// ASRConfigured reports whether audio transcription is minimally configured.
|
||||
func (c Config) ASRConfigured() bool { return c.asrBaseURL() != "" && c.ASRModel != "" }
|
||||
|
||||
// Client is a minimal OpenAI-compatible API client with shared retry/backoff
|
||||
// conventions (429 with Retry-After, 5xx, transport errors, and gateway "200
|
||||
// with embedded error object" responses are retried; other 4xx fail fast).
|
||||
// The config is held behind an atomic pointer so admin settings changes apply
|
||||
// to subsequent requests without rebuilding the client; each request snapshots
|
||||
// the config once so its URL, key, and model stay consistent.
|
||||
type Client struct {
|
||||
cfg atomic.Pointer[Config]
|
||||
// chatHTTP caps a single chat completion at 10 minutes. asrHTTP has no
|
||||
// client-level timeout: a transcription upload's deadline is set per request
|
||||
// (sized to the chunk duration), which a client timeout would silently cap.
|
||||
chatHTTP *http.Client
|
||||
asrHTTP *http.Client
|
||||
}
|
||||
|
||||
// NewClient builds a client from the endpoint config.
|
||||
func NewClient(cfg Config) *Client {
|
||||
c := &Client{
|
||||
chatHTTP: &http.Client{Timeout: 10 * time.Minute},
|
||||
asrHTTP: &http.Client{},
|
||||
}
|
||||
c.UpdateConfig(cfg)
|
||||
return c
|
||||
}
|
||||
|
||||
// UpdateConfig swaps the endpoint config. Safe for concurrent use; in-flight
|
||||
// requests finish with the config they started with.
|
||||
func (c *Client) UpdateConfig(cfg Config) {
|
||||
c.cfg.Store(&cfg)
|
||||
}
|
||||
|
||||
// Config returns a snapshot of the current endpoint config.
|
||||
func (c *Client) Config() Config {
|
||||
return *c.cfg.Load()
|
||||
}
|
||||
|
||||
// Message is one chat-completions message.
|
||||
type Message struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type chatResponseFormat struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
type chatCompletionRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []Message `json:"messages"`
|
||||
Temperature float32 `json:"temperature"`
|
||||
ResponseFormat *chatResponseFormat `json:"response_format,omitempty"`
|
||||
}
|
||||
|
||||
type chatCompletionResponse struct {
|
||||
Choices []struct {
|
||||
Message Message `json:"message"`
|
||||
} `json:"choices"`
|
||||
// Some OpenAI-compatible gateways (e.g. OpenRouter) return a 200 with an
|
||||
// error object instead of an HTTP error status when an upstream provider
|
||||
// fails. We surface and retry on it.
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// Chat performs one chat completion and returns the first choice's content.
|
||||
// When jsonObject is true it requests response_format=json_object; providers
|
||||
// that ignore the field still work because the prompt itself demands JSON.
|
||||
func (c *Client) Chat(ctx context.Context, messages []Message, jsonObject bool) (string, error) {
|
||||
cfg := c.Config()
|
||||
reqBody := chatCompletionRequest{
|
||||
Model: cfg.ChatModel,
|
||||
Messages: messages,
|
||||
Temperature: 0.2,
|
||||
}
|
||||
if jsonObject {
|
||||
reqBody.ResponseFormat = &chatResponseFormat{Type: "json_object"}
|
||||
}
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal chat request: %w", err)
|
||||
}
|
||||
|
||||
url := endpointURL(cfg.BaseURL, "chat/completions")
|
||||
|
||||
var content string
|
||||
err = c.doWithRetry(ctx, c.chatHTTP, "chat API",
|
||||
func() (*http.Request, error) {
|
||||
httpReq, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if reqErr != nil {
|
||||
return nil, fmt.Errorf("create request: %w", reqErr)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if cfg.APIKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey)
|
||||
}
|
||||
return httpReq, nil
|
||||
},
|
||||
func(respBody []byte) error {
|
||||
var parsed chatCompletionResponse
|
||||
if err := json.Unmarshal(respBody, &parsed); err != nil {
|
||||
return fmt.Errorf("decode chat response: %w", err)
|
||||
}
|
||||
if parsed.Error != nil && parsed.Error.Message != "" {
|
||||
// 200 with an upstream error object — transient on gateways.
|
||||
return fmt.Errorf("chat API error: %s", parsed.Error.Message)
|
||||
}
|
||||
if len(parsed.Choices) == 0 || parsed.Choices[0].Message.Content == "" {
|
||||
slog.Warn("AI chat returned no choices, retrying",
|
||||
"model", cfg.ChatModel, "response_bytes", len(respBody))
|
||||
return fmt.Errorf("chat API returned no choices")
|
||||
}
|
||||
content = parsed.Choices[0].Message.Content
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// SystemUserChat performs one chat completion from a system + user prompt
|
||||
// pair, requesting a JSON object response. Its signature matches
|
||||
// aitranslate.ChatFn so a client method reference wires straight in.
|
||||
func (c *Client) SystemUserChat(ctx context.Context, system, user string) (string, error) {
|
||||
return c.Chat(ctx, []Message{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, true)
|
||||
}
|
||||
|
||||
// permanentError marks a parse failure that retrying cannot fix (e.g. an
|
||||
// endpoint that structurally does not support the requested response format).
|
||||
type permanentError struct{ err error }
|
||||
|
||||
func (e *permanentError) Error() string { return e.err.Error() }
|
||||
func (e *permanentError) Unwrap() error { return e.err }
|
||||
|
||||
// doWithRetry runs the shared request/retry loop: build constructs a fresh
|
||||
// request per attempt, parse consumes a 200 body. Transport errors, read
|
||||
// errors, 429 (honoring Retry-After), 5xx, and retryable parse errors back
|
||||
// off and retry; other 4xx and permanentError parse failures return at once.
|
||||
func (c *Client) doWithRetry(ctx context.Context, httpClient *http.Client, label string,
|
||||
build func() (*http.Request, error), parse func(body []byte) error) error {
|
||||
const maxAttempts = 6
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
httpReq, err := build()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, doErr := httpClient.Do(httpReq)
|
||||
if doErr != nil {
|
||||
lastErr = fmt.Errorf("%s request failed: %w", label, doErr)
|
||||
if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil {
|
||||
return waitErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
respBody, readErr := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if readErr != nil {
|
||||
// A truncated/failed read could otherwise be misparsed as a valid
|
||||
// (empty) response; treat it as a retryable transport error.
|
||||
lastErr = fmt.Errorf("read %s response: %w", label, readErr)
|
||||
if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil {
|
||||
return waitErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusTooManyRequests:
|
||||
wait := rateLimitBackoff(resp, attempt)
|
||||
slog.Warn("rate limited by AI API, waiting", "api", label, "attempt", attempt+1, "wait", wait)
|
||||
lastErr = fmt.Errorf("%s returned 429: %s", label, Truncate(string(respBody), 300))
|
||||
if waitErr := sleepCtx(ctx, wait); waitErr != nil {
|
||||
return waitErr
|
||||
}
|
||||
continue
|
||||
case resp.StatusCode >= 500:
|
||||
lastErr = fmt.Errorf("%s returned %d: %s", label, resp.StatusCode, Truncate(string(respBody), 300))
|
||||
if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil {
|
||||
return waitErr
|
||||
}
|
||||
continue
|
||||
case resp.StatusCode != http.StatusOK:
|
||||
// 4xx other than 429: not retryable.
|
||||
return fmt.Errorf("%s returned %d: %s", label, resp.StatusCode, Truncate(string(respBody), 300))
|
||||
}
|
||||
|
||||
parseErr := parse(respBody)
|
||||
if parseErr == nil {
|
||||
return nil
|
||||
}
|
||||
var perm *permanentError
|
||||
if errors.As(parseErr, &perm) {
|
||||
return perm.err
|
||||
}
|
||||
lastErr = parseErr
|
||||
if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil {
|
||||
return waitErr
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("%s: retries exhausted", label)
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// endpointURL joins a configured base URL with an OpenAI API path,
|
||||
// tolerating bases that already include the version segment (e.g.
|
||||
// DeepInfra's https://api.deepinfra.com/v1/openai) alongside bare hosts
|
||||
// (https://api.openai.com) and prefixed hosts (https://api.groq.com/openai).
|
||||
func endpointURL(base, path string) string {
|
||||
base = strings.TrimRight(base, "/")
|
||||
if strings.Contains(base, "/v1") {
|
||||
return base + "/" + path
|
||||
}
|
||||
return base + "/v1/" + path
|
||||
}
|
||||
|
||||
// Truncate caps a string for inclusion in an error or log line.
|
||||
func Truncate(s string, maxLen int) string {
|
||||
if len(s) > maxLen {
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// sleepCtx waits for d or until ctx is cancelled.
|
||||
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(d):
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// rateLimitBackoff returns how long to wait after a 429, honoring Retry-After
|
||||
// and otherwise backing off exponentially (capped at 60s).
|
||||
func rateLimitBackoff(resp *http.Response, attempt int) time.Duration {
|
||||
if ra := resp.Header.Get("Retry-After"); ra != "" {
|
||||
if secs, err := strconv.Atoi(ra); err == nil && secs > 0 {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
}
|
||||
wait := 10 * time.Second * (1 << attempt)
|
||||
if wait > 60*time.Second {
|
||||
wait = 60 * time.Second
|
||||
}
|
||||
return wait
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func chatConfig(baseURL string) Config {
|
||||
return Config{BaseURL: baseURL, APIKey: "test-key", ChatModel: "test-model"}
|
||||
}
|
||||
|
||||
const chatOK = `{"choices":[{"message":{"role":"assistant","content":"hello"}}]}`
|
||||
|
||||
func TestChatSuccessSendsAuthAndModel(t *testing.T) {
|
||||
var gotAuth, gotBody string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
buf := make([]byte, 4096)
|
||||
n, _ := r.Body.Read(buf)
|
||||
gotBody = string(buf[:n])
|
||||
w.Write([]byte(chatOK))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient(chatConfig(srv.URL))
|
||||
out, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Chat: %v", err)
|
||||
}
|
||||
if out != "hello" {
|
||||
t.Errorf("content = %q, want hello", out)
|
||||
}
|
||||
if gotAuth != "Bearer test-key" {
|
||||
t.Errorf("auth = %q", gotAuth)
|
||||
}
|
||||
if !strings.Contains(gotBody, `"model":"test-model"`) || !strings.Contains(gotBody, `"json_object"`) {
|
||||
t.Errorf("request body missing model/response_format: %s", gotBody)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatRetriesOn429HonoringRetryAfter(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if calls.Add(1) == 1 {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
w.Write([]byte(chatOK))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
start := time.Now()
|
||||
c := NewClient(chatConfig(srv.URL))
|
||||
if _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, false); err != nil {
|
||||
t.Fatalf("Chat: %v", err)
|
||||
}
|
||||
if calls.Load() != 2 {
|
||||
t.Errorf("calls = %d, want 2", calls.Load())
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < time.Second {
|
||||
t.Errorf("did not honor Retry-After: elapsed %v", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatRetriesOn5xxAndEmbeddedErrorAndEmptyChoices(t *testing.T) {
|
||||
responses := []func(w http.ResponseWriter){
|
||||
func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadGateway) },
|
||||
func(w http.ResponseWriter) { w.Write([]byte(`{"error":{"message":"upstream sad"}}`)) },
|
||||
func(w http.ResponseWriter) { w.Write([]byte(`{"choices":[]}`)) },
|
||||
func(w http.ResponseWriter) { w.Write([]byte(chatOK)) },
|
||||
}
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
responses[calls.Add(1)-1](w)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient(chatConfig(srv.URL))
|
||||
out, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("Chat: %v", err)
|
||||
}
|
||||
if out != "hello" || calls.Load() != 4 {
|
||||
t.Errorf("out=%q calls=%d, want hello/4", out, calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatFailsFastOnNon429ClientError(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient(chatConfig(srv.URL))
|
||||
if _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, false); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Errorf("calls = %d, want 1 (no retry on 401)", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
const verboseJSON = `{"language":"english","text":"hi there","segments":[{"start":0.0,"end":1.5,"text":" hi"},{"start":1.5,"end":3.0,"text":" there"}]}`
|
||||
|
||||
func TestTranscribeParsesSegmentsAndMultipart(t *testing.T) {
|
||||
var gotModel, gotFormat, gotLang, gotAuth, gotVAD string
|
||||
var gotGranularities []string
|
||||
var gotFile []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
t.Errorf("parse multipart: %v", err)
|
||||
}
|
||||
gotModel = r.FormValue("model")
|
||||
gotFormat = r.FormValue("response_format")
|
||||
gotLang = r.FormValue("language")
|
||||
gotVAD = r.FormValue("vad_filter")
|
||||
gotGranularities = r.MultipartForm.Value["timestamp_granularities[]"]
|
||||
f, _, err := r.FormFile("file")
|
||||
if err == nil {
|
||||
buf := make([]byte, 64)
|
||||
n, _ := f.Read(buf)
|
||||
gotFile = buf[:n]
|
||||
f.Close()
|
||||
}
|
||||
w.Write([]byte(verboseJSON))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := chatConfig(srv.URL)
|
||||
cfg.ASRModel = "whisper-test"
|
||||
c := NewClient(cfg)
|
||||
tr, err := c.Transcribe(context.Background(), TranscribeRequest{
|
||||
Filename: "chunk.wav", Audio: []byte("RIFFfake"), Language: "ja",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if gotModel != "whisper-test" || gotFormat != "verbose_json" || gotLang != "ja" {
|
||||
t.Errorf("fields model=%q format=%q lang=%q", gotModel, gotFormat, gotLang)
|
||||
}
|
||||
// Local (non-hosted) endpoint: faster-whisper VAD must be requested or
|
||||
// segment timestamps stretch wall-to-wall across silence.
|
||||
if gotVAD != "true" {
|
||||
t.Errorf("vad_filter = %q, want true for a self-hosted endpoint", gotVAD)
|
||||
}
|
||||
if fmt.Sprint(gotGranularities) != "[segment word]" {
|
||||
t.Errorf("timestamp_granularities[] = %v, want [segment word]", gotGranularities)
|
||||
}
|
||||
if gotAuth != "Bearer test-key" {
|
||||
t.Errorf("auth = %q (should fall back to chat key)", gotAuth)
|
||||
}
|
||||
if string(gotFile) != "RIFFfake" {
|
||||
t.Errorf("file payload = %q", gotFile)
|
||||
}
|
||||
if tr.Language != "english" || len(tr.Segments) != 2 || tr.Segments[1].Text != " there" || tr.Segments[1].End != 3.0 {
|
||||
t.Errorf("unexpected transcription: %+v", tr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeOmitsVADFilterForStrictHostedEndpoints(t *testing.T) {
|
||||
for url, wantVAD := range map[string]bool{
|
||||
"https://api.openai.com": false,
|
||||
"https://myorg.openai.azure.com": false,
|
||||
"https://api.groq.com/openai": false,
|
||||
"http://192.168.1.10:8000": true,
|
||||
"https://whisper.example.com": true,
|
||||
"https://api.deepinfra.com/v1/openai": true,
|
||||
} {
|
||||
got := !hostMatchesAny(url, strictHostedASRHosts)
|
||||
if got != wantVAD {
|
||||
t.Errorf("vad_filter for %s = %v, want %v", url, got, wantVAD)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeParsesPerSegmentWords(t *testing.T) {
|
||||
// speaches/faster-whisper shape: words nested inside each segment.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"language":"en","text":"hi there","segments":[
|
||||
{"start":0.0,"end":3.0,"text":" hi there","words":[
|
||||
{"start":0.2,"end":0.5,"word":" hi"},{"start":0.6,"end":1.0,"word":" there"}]}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := chatConfig(srv.URL)
|
||||
cfg.ASRModel = "whisper-test"
|
||||
tr, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")})
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
words := tr.Segments[0].Words
|
||||
if len(words) != 2 || words[1].Text != " there" || words[1].Start != 0.6 || words[1].End != 1.0 {
|
||||
t.Errorf("segment words = %+v", words)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeAttachesTopLevelWordsBySegmentTime(t *testing.T) {
|
||||
// OpenAI shape: words in a top-level array, segments without words.
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"language":"en","text":"hi there friend","segments":[
|
||||
{"start":0.0,"end":1.5,"text":" hi there"},{"start":1.5,"end":3.0,"text":" friend"}],
|
||||
"words":[{"start":0.2,"end":0.5,"word":"hi"},{"start":0.6,"end":1.0,"word":"there"},
|
||||
{"start":1.8,"end":2.2,"word":"friend"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := chatConfig(srv.URL)
|
||||
cfg.ASRModel = "whisper-test"
|
||||
tr, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")})
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if len(tr.Segments[0].Words) != 2 || len(tr.Segments[1].Words) != 1 {
|
||||
t.Errorf("word distribution = %d/%d, want 2/1", len(tr.Segments[0].Words), len(tr.Segments[1].Words))
|
||||
}
|
||||
if tr.Segments[1].Words[0].Text != "friend" {
|
||||
t.Errorf("segment 1 word = %+v", tr.Segments[1].Words[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeEmptySegmentsIsNotAnError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"language":"english","text":"","segments":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := chatConfig(srv.URL)
|
||||
cfg.ASRModel = "whisper-test"
|
||||
tr, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")})
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if len(tr.Segments) != 0 {
|
||||
t.Errorf("segments = %v, want empty", tr.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeMissingSegmentsFieldFailsFast(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
w.Write([]byte(`{"text":"plain response without segments"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := chatConfig(srv.URL)
|
||||
cfg.ASRModel = "whisper-test"
|
||||
_, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")})
|
||||
if err == nil || !strings.Contains(err.Error(), "verbose_json") {
|
||||
t.Fatalf("err = %v, want verbose_json complaint", err)
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Errorf("calls = %d, want 1 (permanent error must not retry)", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeUsesASROverrides(t *testing.T) {
|
||||
var gotAuth atomic.Value
|
||||
asrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth.Store(r.Header.Get("Authorization"))
|
||||
w.Write([]byte(verboseJSON))
|
||||
}))
|
||||
defer asrSrv.Close()
|
||||
|
||||
cfg := Config{
|
||||
BaseURL: "http://chat.invalid", APIKey: "chat-key", ChatModel: "m",
|
||||
ASRBaseURL: asrSrv.URL, ASRAPIKey: "asr-key", ASRModel: "whisper-test",
|
||||
}
|
||||
if _, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if gotAuth.Load() != "Bearer asr-key" {
|
||||
t.Errorf("auth = %q, want asr-key", gotAuth.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeRequiresConfig(t *testing.T) {
|
||||
c := NewClient(Config{BaseURL: "http://x", ChatModel: "m"}) // no ASR model
|
||||
if _, err := c.Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}); err == nil {
|
||||
t.Fatal("expected not-configured error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeChatOnlyGatewayGetsConfigHint(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
w.Write([]byte(`{"error":{"message":"invalid content-type: multipart/form-data","code":400}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := chatConfig(srv.URL)
|
||||
cfg.ASRModel = "whisper-test"
|
||||
_, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")})
|
||||
if err == nil || !strings.Contains(err.Error(), "Whisper-compatible Transcription base URL") {
|
||||
t.Fatalf("err = %v, want configuration hint", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsChatOnlyGateway(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"https://openrouter.ai/api": true,
|
||||
"https://openrouter.ai": true,
|
||||
"openrouter.ai/api/v1": true,
|
||||
"https://gateway.openrouter.ai": true,
|
||||
"https://api.groq.com/openai": false,
|
||||
"https://api.openai.com": false,
|
||||
"http://localhost:8000": false,
|
||||
"https://my-openrouter.ai.example.io": false,
|
||||
"": false,
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := IsChatOnlyGateway(in); got != want {
|
||||
t.Errorf("IsChatOnlyGateway(%q) = %v, want %v", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeRefusesChatOnlyGateway(t *testing.T) {
|
||||
cfg := Config{BaseURL: "https://openrouter.ai/api", ChatModel: "m", ASRModel: "whisper-1"}
|
||||
_, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")})
|
||||
if err == nil || !strings.Contains(err.Error(), "cannot produce timestamped transcriptions") {
|
||||
t.Fatalf("err = %v, want chat-only refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndpointURLToleratesVersionedBases(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"https://api.openai.com": "https://api.openai.com/v1/chat/completions",
|
||||
"https://api.groq.com/openai": "https://api.groq.com/openai/v1/chat/completions",
|
||||
"https://api.deepinfra.com/v1/openai": "https://api.deepinfra.com/v1/openai/chat/completions",
|
||||
"http://localhost:8969": "http://localhost:8969/v1/chat/completions",
|
||||
"http://localhost:8969/v1": "http://localhost:8969/v1/chat/completions",
|
||||
"https://api.openai.com/": "https://api.openai.com/v1/chat/completions",
|
||||
}
|
||||
for base, want := range cases {
|
||||
if got := endpointURL(base, "chat/completions"); got != want {
|
||||
t.Errorf("endpointURL(%q) = %q, want %q", base, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// defaultTranscribeTimeout bounds a single transcription request when the
|
||||
// caller does not size one to its chunk duration. Generous on purpose: local
|
||||
// Whisper servers on modest hardware can run well below realtime.
|
||||
const defaultTranscribeTimeout = 20 * time.Minute
|
||||
|
||||
// TranscribeRequest is one audio-transcription call. Audio is held in memory
|
||||
// so the request can be rebuilt across retries; callers chunk long files
|
||||
// (a 10-minute 16 kHz mono WAV is ~19 MB).
|
||||
type TranscribeRequest struct {
|
||||
Filename string // e.g. "chunk00001.wav"; the extension hints the container
|
||||
Audio []byte
|
||||
Language string // optional ISO-639-1 hint; empty lets the model detect
|
||||
Timeout time.Duration // per-request deadline; 0 uses defaultTranscribeTimeout
|
||||
}
|
||||
|
||||
// TranscriptionSegment is one timed segment of recognized speech, in seconds
|
||||
// relative to the start of the submitted audio.
|
||||
type TranscriptionSegment struct {
|
||||
Start float64
|
||||
End float64
|
||||
Text string
|
||||
// Words carries per-word timings when the endpoint honors
|
||||
// timestamp_granularities[]=word; empty otherwise. Word-level times let
|
||||
// cue building split paragraph-length segments and end cues when speech
|
||||
// actually stops instead of when the next segment begins.
|
||||
Words []TranscriptionWord
|
||||
}
|
||||
|
||||
// TranscriptionWord is one recognized word with timing, in seconds relative
|
||||
// to the start of the submitted audio.
|
||||
type TranscriptionWord struct {
|
||||
Start float64
|
||||
End float64
|
||||
Text string
|
||||
}
|
||||
|
||||
// Transcription is the parsed verbose_json transcription result.
|
||||
type Transcription struct {
|
||||
// Language is the detected (or hinted) language as reported by the
|
||||
// endpoint. OpenAI returns an English language name ("english"); other
|
||||
// servers return ISO codes. Callers must normalize.
|
||||
Language string
|
||||
// Segments may legitimately be empty for speech-free audio (silence,
|
||||
// music-only chunks).
|
||||
Segments []TranscriptionSegment
|
||||
}
|
||||
|
||||
type transcriptionWordJSON struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Word string `json:"word"`
|
||||
}
|
||||
|
||||
type transcriptionResponse struct {
|
||||
Language string `json:"language"`
|
||||
Text string `json:"text"`
|
||||
// Segments distinguishes "verbose_json honored, no speech" (empty array)
|
||||
// from "endpoint ignored verbose_json" (field absent) — the latter cannot
|
||||
// produce timed cues and must fail rather than silently emit nothing.
|
||||
Segments *[]struct {
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Words []transcriptionWordJSON `json:"words"`
|
||||
} `json:"segments"`
|
||||
// Words is where OpenAI puts word timings; faster-whisper servers
|
||||
// (speaches) nest them inside each segment instead.
|
||||
Words []transcriptionWordJSON `json:"words"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// chatOnlyGatewayHosts lists OpenAI-compatible gateways that cannot serve
|
||||
// Silo's transcription needs: they either lack /v1/audio/transcriptions
|
||||
// entirely or (OpenRouter) return plain text with no segment timestamps,
|
||||
// which subtitle cues require. Matched by host suffix.
|
||||
var chatOnlyGatewayHosts = []string{
|
||||
"openrouter.ai",
|
||||
}
|
||||
|
||||
// strictHostedASRHosts lists hosted transcription providers that reject
|
||||
// multipart fields outside the OpenAI spec, so the faster-whisper-only
|
||||
// vad_filter field must be omitted for them. Nothing is lost: hosted Whisper
|
||||
// runs voice-activity detection server-side. Self-hosted servers (speaches,
|
||||
// faster-whisper) need the explicit field — without it segment timestamps
|
||||
// stretch wall-to-wall across silence. Matched by host suffix.
|
||||
var strictHostedASRHosts = []string{
|
||||
"api.openai.com",
|
||||
"openai.azure.com",
|
||||
"api.groq.com",
|
||||
"api.mistral.ai",
|
||||
}
|
||||
|
||||
// IsChatOnlyGateway reports whether baseURL points at a known chat-only
|
||||
// gateway that cannot produce timestamped transcriptions. Used to validate
|
||||
// the transcription settings and to disable ASR rather than fail jobs at
|
||||
// runtime.
|
||||
func IsChatOnlyGateway(baseURL string) bool {
|
||||
return hostMatchesAny(baseURL, chatOnlyGatewayHosts)
|
||||
}
|
||||
|
||||
// hostMatchesAny reports whether baseURL's hostname equals or is a subdomain
|
||||
// of any of the given host suffixes.
|
||||
func hostMatchesAny(baseURL string, hosts []string) bool {
|
||||
baseURL = strings.TrimSpace(baseURL)
|
||||
if baseURL == "" {
|
||||
return false
|
||||
}
|
||||
if !strings.Contains(baseURL, "://") {
|
||||
baseURL = "https://" + baseURL
|
||||
}
|
||||
u, err := url.Parse(baseURL)
|
||||
if err != nil || u.Host == "" {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
for _, h := range hosts {
|
||||
if host == h || strings.HasSuffix(host, "."+h) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Transcribe performs one audio transcription against the ASR endpoint
|
||||
// (falling back to the chat endpoint's base URL/key when no ASR override is
|
||||
// configured), using the OpenAI-compatible /v1/audio/transcriptions API with
|
||||
// response_format=verbose_json for segment timestamps.
|
||||
func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (*Transcription, error) {
|
||||
cfg := c.Config()
|
||||
if !cfg.ASRConfigured() {
|
||||
return nil, fmt.Errorf("transcription endpoint is not configured")
|
||||
}
|
||||
if IsChatOnlyGateway(cfg.asrBaseURL()) {
|
||||
return nil, fmt.Errorf("the configured transcription endpoint (%s) cannot produce timestamped transcriptions; "+
|
||||
"set a Whisper-compatible Transcription base URL under Admin Settings → AI Services", cfg.asrBaseURL())
|
||||
}
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("no audio data to transcribe")
|
||||
}
|
||||
|
||||
timeout := req.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultTranscribeTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
url := endpointURL(cfg.asrBaseURL(), "audio/transcriptions")
|
||||
|
||||
var result *Transcription
|
||||
doErr := c.doWithRetry(ctx, c.asrHTTP, "transcription API",
|
||||
func() (*http.Request, error) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile("file", req.Filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create multipart file: %w", err)
|
||||
}
|
||||
if _, err := fw.Write(req.Audio); err != nil {
|
||||
return nil, fmt.Errorf("write multipart audio: %w", err)
|
||||
}
|
||||
fields := [][2]string{
|
||||
{"model", cfg.ASRModel},
|
||||
{"response_format", "verbose_json"},
|
||||
{"temperature", "0"},
|
||||
// Word timings let cue building split paragraph-length
|
||||
// segments and end cues when speech actually stops; segment
|
||||
// granularity must be requested alongside or OpenAI omits it.
|
||||
{"timestamp_granularities[]", "segment"},
|
||||
{"timestamp_granularities[]", "word"},
|
||||
}
|
||||
if req.Language != "" {
|
||||
fields = append(fields, [2]string{"language", req.Language})
|
||||
}
|
||||
if !hostMatchesAny(cfg.asrBaseURL(), strictHostedASRHosts) {
|
||||
fields = append(fields, [2]string{"vad_filter", "true"})
|
||||
}
|
||||
for _, f := range fields {
|
||||
if err := w.WriteField(f[0], f[1]); err != nil {
|
||||
return nil, fmt.Errorf("write multipart field %s: %w", f[0], err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("finalize multipart body: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf.Bytes()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", w.FormDataContentType())
|
||||
if key := cfg.asrAPIKey(); key != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+key)
|
||||
}
|
||||
return httpReq, nil
|
||||
},
|
||||
func(respBody []byte) error {
|
||||
var parsed transcriptionResponse
|
||||
if err := json.Unmarshal(respBody, &parsed); err != nil {
|
||||
return fmt.Errorf("decode transcription response: %w", err)
|
||||
}
|
||||
if parsed.Error != nil && parsed.Error.Message != "" {
|
||||
return fmt.Errorf("transcription API error: %s", parsed.Error.Message)
|
||||
}
|
||||
if parsed.Segments == nil {
|
||||
return &permanentError{err: fmt.Errorf(
|
||||
"transcription endpoint did not return verbose_json segments (model %q); a verbose_json-capable Whisper endpoint is required", cfg.ASRModel)}
|
||||
}
|
||||
out := &Transcription{Language: parsed.Language}
|
||||
for _, s := range *parsed.Segments {
|
||||
out.Segments = append(out.Segments, TranscriptionSegment{
|
||||
Start: s.Start, End: s.End, Text: s.Text, Words: wordsFromJSON(s.Words),
|
||||
})
|
||||
}
|
||||
attachTopLevelWords(out.Segments, wordsFromJSON(parsed.Words))
|
||||
result = out
|
||||
return nil
|
||||
})
|
||||
if doErr != nil {
|
||||
return nil, decorateTranscribeError(doErr)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func wordsFromJSON(words []transcriptionWordJSON) []TranscriptionWord {
|
||||
out := make([]TranscriptionWord, 0, len(words))
|
||||
for _, w := range words {
|
||||
out = append(out, TranscriptionWord{Start: w.Start, End: w.End, Text: w.Word})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// attachTopLevelWords distributes an OpenAI-style top-level word list onto
|
||||
// segments by word midpoint, for endpoints that report words separately from
|
||||
// segments. Segments that already carry their own words are left untouched.
|
||||
func attachTopLevelWords(segments []TranscriptionSegment, words []TranscriptionWord) {
|
||||
if len(segments) == 0 || len(words) == 0 {
|
||||
return
|
||||
}
|
||||
for _, s := range segments {
|
||||
if len(s.Words) > 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
si := 0
|
||||
for _, w := range words {
|
||||
mid := (w.Start + w.End) / 2
|
||||
for si < len(segments)-1 && mid >= segments[si].End {
|
||||
si++
|
||||
}
|
||||
segments[si].Words = append(segments[si].Words, w)
|
||||
}
|
||||
}
|
||||
|
||||
// decorateTranscribeError appends a configuration hint to the errors a
|
||||
// chat-only gateway produces when it receives a transcription upload (no
|
||||
// /v1/audio/transcriptions route, or multipart rejected). Operators routinely
|
||||
// point the shared base URL at a chat-only provider; without the hint the raw
|
||||
// 400/404 reads like a pipeline bug instead of "set a Whisper endpoint".
|
||||
func decorateTranscribeError(err error) error {
|
||||
msg := err.Error()
|
||||
likelyUnsupported := strings.Contains(msg, "returned 400") ||
|
||||
strings.Contains(msg, "returned 404") ||
|
||||
strings.Contains(msg, "returned 405")
|
||||
if !likelyUnsupported {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w — the configured endpoint likely does not support audio transcription; "+
|
||||
"set a Whisper-compatible Transcription base URL (and model) under Admin Settings → AI Services "+
|
||||
"(e.g. a self-hosted faster-whisper/speaches server, api.groq.com/openai with whisper-large-v3-turbo, or api.openai.com with whisper-1)", err)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package aitranslate
|
||||
|
||||
import "strings"
|
||||
|
||||
// languageNames maps common ISO 639-1 codes to English names. The model handles
|
||||
// bare codes acceptably, but full names noticeably improve translation quality,
|
||||
// so we resolve the common cases and fall back to the raw code otherwise.
|
||||
var languageNames = map[string]string{
|
||||
"ar": "Arabic", "bg": "Bulgarian", "bn": "Bengali", "cs": "Czech",
|
||||
"da": "Danish", "de": "German", "el": "Greek", "en": "English",
|
||||
"es": "Spanish", "et": "Estonian", "fa": "Persian", "fi": "Finnish",
|
||||
"fr": "French", "he": "Hebrew", "hi": "Hindi", "hr": "Croatian",
|
||||
"hu": "Hungarian", "id": "Indonesian", "it": "Italian", "ja": "Japanese",
|
||||
"ko": "Korean", "lt": "Lithuanian", "lv": "Latvian", "ms": "Malay",
|
||||
"nl": "Dutch", "no": "Norwegian", "pl": "Polish", "pt": "Portuguese",
|
||||
"ro": "Romanian", "ru": "Russian", "sk": "Slovak", "sl": "Slovenian",
|
||||
"sr": "Serbian", "sv": "Swedish", "ta": "Tamil", "th": "Thai",
|
||||
"tr": "Turkish", "uk": "Ukrainian", "vi": "Vietnamese", "zh": "Chinese",
|
||||
}
|
||||
|
||||
// LanguageCodeFromName maps an English language name back to its ISO 639-1
|
||||
// code — Whisper endpoints report detected languages as names ("english")
|
||||
// while local servers report codes. Returns "" when unknown.
|
||||
func LanguageCodeFromName(name string) string {
|
||||
name = strings.ToLower(strings.TrimSpace(name))
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
for code, display := range languageNames {
|
||||
if strings.ToLower(display) == name {
|
||||
return code
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// LanguageDisplayName returns a human-readable language name for a code, or
|
||||
// the trimmed code itself when unknown. An empty code yields an empty string.
|
||||
func LanguageDisplayName(code string) string {
|
||||
code = strings.TrimSpace(code)
|
||||
if code == "" {
|
||||
return ""
|
||||
}
|
||||
base := strings.ToLower(code)
|
||||
if i := strings.IndexAny(base, "-_"); i >= 0 {
|
||||
base = base[:i]
|
||||
}
|
||||
if name, ok := languageNames[base]; ok {
|
||||
return name
|
||||
}
|
||||
return code
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
// Package aitranslate implements generic batched text translation over an
|
||||
// OpenAI-compatible chat model. Callers hand it ordered text segments and a
|
||||
// domain-specific system prompt; it owns the wire protocol (indexed-JSON
|
||||
// batches the model must echo back key-for-key), context carryover between
|
||||
// batches, and retries on malformed model output. Subtitle cues and metadata
|
||||
// descriptions both translate through this one implementation.
|
||||
package aitranslate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Segment is one unit of translatable text. ID is the caller's identifier and
|
||||
// is never sent to the model (batches are keyed by position), so any stable
|
||||
// string works.
|
||||
type Segment struct {
|
||||
ID string
|
||||
Text string
|
||||
}
|
||||
|
||||
// ChatFn performs one chat completion. Wiring this as a function keeps the
|
||||
// package independent of the client and trivially testable.
|
||||
type ChatFn func(ctx context.Context, system, user string) (string, error)
|
||||
|
||||
// Request describes one translation run.
|
||||
type Request struct {
|
||||
Segments []Segment
|
||||
SystemPrompt string // domain-specific; must demand same-keys JSON output
|
||||
// TargetName is the human-readable target language name used in the
|
||||
// per-batch user message.
|
||||
TargetName string
|
||||
// EntryNoun names the units in the user message ("cues", "descriptions");
|
||||
// empty defaults to "entries".
|
||||
EntryNoun string
|
||||
// BatchSize is the number of segments per chat request (required, > 0).
|
||||
BatchSize int
|
||||
// ContextNeighbors is how many preceding source segments are included,
|
||||
// untranslated, for continuity across batch boundaries.
|
||||
ContextNeighbors int
|
||||
}
|
||||
|
||||
// maxFormatRetries is how many times a batch is re-asked after the model
|
||||
// returns unparsable or incomplete JSON. Transport/API errors are retried
|
||||
// inside the chat client and are not part of this count.
|
||||
const maxFormatRetries = 2
|
||||
|
||||
// Translate translates req.Segments in order, preserving count and IDs and
|
||||
// only rewriting Text. onBatch, when non-nil, is called after each batch with
|
||||
// that batch's translated segments and overall progress (done/total
|
||||
// segments). Implementations must honor ctx cancellation between batches.
|
||||
func Translate(ctx context.Context, chat ChatFn, req Request, onBatch func(batch []Segment, done, total int)) ([]Segment, error) {
|
||||
if chat == nil {
|
||||
return nil, fmt.Errorf("chat function is nil")
|
||||
}
|
||||
if req.BatchSize <= 0 {
|
||||
return nil, fmt.Errorf("invalid batch size: %d", req.BatchSize)
|
||||
}
|
||||
total := len(req.Segments)
|
||||
if total == 0 {
|
||||
return nil, fmt.Errorf("no segments to translate")
|
||||
}
|
||||
noun := req.EntryNoun
|
||||
if noun == "" {
|
||||
noun = "entries"
|
||||
}
|
||||
|
||||
// Preserve IDs by copying input segments and only replacing Text.
|
||||
out := make([]Segment, total)
|
||||
copy(out, req.Segments)
|
||||
|
||||
for start := 0; start < total; start += req.BatchSize {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
end := min(start+req.BatchSize, total)
|
||||
|
||||
contextStart := max(0, start-req.ContextNeighbors)
|
||||
translated, err := translateBatch(ctx, chat, req, noun, req.Segments[contextStart:start], req.Segments[start:end])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("translate %s %d-%d: %w", noun, start+1, end, err)
|
||||
}
|
||||
for i, text := range translated {
|
||||
out[start+i].Text = text
|
||||
}
|
||||
if onBatch != nil {
|
||||
onBatch(out[start:end], end, total)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func translateBatch(ctx context.Context, chat ChatFn, req Request, noun string, contextSegs, batch []Segment) ([]string, error) {
|
||||
texts := make([]string, len(batch))
|
||||
for i, s := range batch {
|
||||
texts[i] = s.Text
|
||||
}
|
||||
payload, err := buildIndexedJSON(texts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var user strings.Builder
|
||||
if len(contextSegs) > 0 {
|
||||
user.WriteString("Preceding lines for context only — do not translate or include them in your output:\n")
|
||||
for _, s := range contextSegs {
|
||||
user.WriteString(strings.ReplaceAll(s.Text, "\n", " "))
|
||||
user.WriteByte('\n')
|
||||
}
|
||||
user.WriteByte('\n')
|
||||
}
|
||||
target := req.TargetName
|
||||
if target == "" {
|
||||
target = "the target language"
|
||||
}
|
||||
fmt.Fprintf(&user, "Translate these %d %s into %s. Respond with only the JSON object:\n%s", len(batch), noun, target, payload)
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxFormatRetries; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content, err := chat(ctx, req.SystemPrompt, user.String())
|
||||
if err != nil {
|
||||
return nil, err // transport/API errors are already retried inside the client
|
||||
}
|
||||
|
||||
obj, err := extractJSONObject(content)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(obj), &m); err != nil {
|
||||
lastErr = fmt.Errorf("decode translation JSON: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
out := make([]string, len(batch))
|
||||
complete := true
|
||||
for i := range batch {
|
||||
v, ok := m[strconv.Itoa(i+1)]
|
||||
if !ok {
|
||||
complete = false
|
||||
break
|
||||
}
|
||||
out[i] = v
|
||||
}
|
||||
if !complete {
|
||||
lastErr = fmt.Errorf("model omitted one or more %s", noun)
|
||||
continue
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("invalid model response after %d attempts: %w", maxFormatRetries+1, lastErr)
|
||||
}
|
||||
|
||||
// buildIndexedJSON renders texts as a JSON object {"1":..., "2":...} keyed by
|
||||
// 1-based position, escaping each value safely. It is built by hand rather
|
||||
// than json.Marshal'ing a map so the keys stay in numeric order — that reads
|
||||
// more naturally for the model than the lexicographic order Go emits for maps
|
||||
// ("1","10","11",...,"2"). Correctness doesn't depend on order (results are
|
||||
// mapped back by key), but ordered input gives the model better context.
|
||||
func buildIndexedJSON(texts []string) (string, error) {
|
||||
var b strings.Builder
|
||||
b.WriteByte('{')
|
||||
for i, text := range texts {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
key, err := json.Marshal(strconv.Itoa(i + 1))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
val, err := json.Marshal(text)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b.Write(key)
|
||||
b.WriteByte(':')
|
||||
b.Write(val)
|
||||
}
|
||||
b.WriteByte('}')
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// extractJSONObject pulls the first balanced-looking JSON object out of a
|
||||
// model reply, tolerating ``` code fences and surrounding prose.
|
||||
func extractJSONObject(s string) (string, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.HasPrefix(s, "```") {
|
||||
s = strings.TrimPrefix(s, "```")
|
||||
if nl := strings.IndexByte(s, '\n'); nl >= 0 {
|
||||
s = s[nl+1:]
|
||||
}
|
||||
if idx := strings.LastIndex(s, "```"); idx >= 0 {
|
||||
s = s[:idx]
|
||||
}
|
||||
s = strings.TrimSpace(s)
|
||||
}
|
||||
start := strings.IndexByte(s, '{')
|
||||
end := strings.LastIndexByte(s, '}')
|
||||
if start < 0 || end < 0 || end < start {
|
||||
return "", fmt.Errorf("no JSON object found in model response")
|
||||
}
|
||||
return s[start : end+1], nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package aitranslate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// echoChat translates by upper-casing every value of the indexed JSON payload
|
||||
// in the user message, mimicking a well-behaved model.
|
||||
func echoChat(t *testing.T) ChatFn {
|
||||
t.Helper()
|
||||
return func(_ context.Context, _ string, user string) (string, error) {
|
||||
obj, err := extractJSONObject(user)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("test chat: %w", err)
|
||||
}
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(obj), &m); err != nil {
|
||||
return "", fmt.Errorf("test chat decode: %w", err)
|
||||
}
|
||||
out := make(map[string]string, len(m))
|
||||
for k, v := range m {
|
||||
out[k] = strings.ToUpper(v)
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
return string(b), nil
|
||||
}
|
||||
}
|
||||
|
||||
func segments(n int) []Segment {
|
||||
out := make([]Segment, n)
|
||||
for i := range out {
|
||||
out[i] = Segment{ID: strconv.Itoa(i + 1), Text: fmt.Sprintf("line %d", i+1)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTranslateBatchesAndPreservesIDs(t *testing.T) {
|
||||
var batches []int
|
||||
out, err := Translate(context.Background(), echoChat(t), Request{
|
||||
Segments: segments(7),
|
||||
BatchSize: 3,
|
||||
}, func(batch []Segment, done, total int) {
|
||||
batches = append(batches, len(batch))
|
||||
if total != 7 {
|
||||
t.Errorf("total = %d, want 7", total)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Translate: %v", err)
|
||||
}
|
||||
if len(out) != 7 {
|
||||
t.Fatalf("len(out) = %d", len(out))
|
||||
}
|
||||
for i, s := range out {
|
||||
if s.ID != strconv.Itoa(i+1) {
|
||||
t.Errorf("ID[%d] = %q, want %d", i, s.ID, i+1)
|
||||
}
|
||||
if want := strings.ToUpper(fmt.Sprintf("line %d", i+1)); s.Text != want {
|
||||
t.Errorf("Text[%d] = %q, want %q", i, s.Text, want)
|
||||
}
|
||||
}
|
||||
if fmt.Sprint(batches) != "[3 3 1]" {
|
||||
t.Errorf("batch sizes = %v, want [3 3 1]", batches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateSendsContextNeighborsUntranslated(t *testing.T) {
|
||||
var secondBatchMsg string
|
||||
call := 0
|
||||
chat := func(ctx context.Context, system, user string) (string, error) {
|
||||
call++
|
||||
if call == 2 {
|
||||
secondBatchMsg = user
|
||||
}
|
||||
return echoChat(t)(ctx, system, user)
|
||||
}
|
||||
_, err := Translate(context.Background(), chat, Request{
|
||||
Segments: segments(4),
|
||||
BatchSize: 2,
|
||||
ContextNeighbors: 2,
|
||||
TargetName: "French",
|
||||
EntryNoun: "cues",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Translate: %v", err)
|
||||
}
|
||||
if !strings.Contains(secondBatchMsg, "context only") ||
|
||||
!strings.Contains(secondBatchMsg, "line 1\nline 2") {
|
||||
t.Errorf("second batch missing context block:\n%s", secondBatchMsg)
|
||||
}
|
||||
if !strings.Contains(secondBatchMsg, "Translate these 2 cues into French.") {
|
||||
t.Errorf("second batch missing instruction:\n%s", secondBatchMsg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateToleratesCodeFences(t *testing.T) {
|
||||
chat := func(_ context.Context, _, _ string) (string, error) {
|
||||
return "Sure! Here you go:\n```json\n{\"1\":\"ok\"}\n```", nil
|
||||
}
|
||||
out, err := Translate(context.Background(), chat, Request{
|
||||
Segments: segments(1),
|
||||
BatchSize: 5,
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Translate: %v", err)
|
||||
}
|
||||
if out[0].Text != "ok" {
|
||||
t.Errorf("Text = %q", out[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateRetriesOmittedKeysThenFails(t *testing.T) {
|
||||
calls := 0
|
||||
chat := func(_ context.Context, _, _ string) (string, error) {
|
||||
calls++
|
||||
return `{"1":"only one"}`, nil // always omits key "2"
|
||||
}
|
||||
_, err := Translate(context.Background(), chat, Request{
|
||||
Segments: segments(2),
|
||||
BatchSize: 5,
|
||||
}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "omitted") {
|
||||
t.Fatalf("err = %v, want omission failure", err)
|
||||
}
|
||||
if calls != maxFormatRetries+1 {
|
||||
t.Errorf("calls = %d, want %d", calls, maxFormatRetries+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslateStopsOnCancelledContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
calls := 0
|
||||
chat := func(_ context.Context, _, _ string) (string, error) {
|
||||
calls++
|
||||
cancel() // cancel after the first batch completes
|
||||
return `{"1":"a"}`, nil
|
||||
}
|
||||
_, err := Translate(ctx, chat, Request{
|
||||
Segments: segments(3),
|
||||
BatchSize: 1,
|
||||
}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "context canceled") {
|
||||
t.Fatalf("err = %v, want context cancellation", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("calls = %d, want 1", calls)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/autoscan"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
mediarequests "github.com/Silo-Server/silo-server/internal/requests"
|
||||
"github.com/Silo-Server/silo-server/internal/scantrigger"
|
||||
)
|
||||
|
||||
// autoscanQueuer is the scan-enqueue surface BuildAutoscanService needs; it is
|
||||
// satisfied by *scanqueue.Service (autoscan.Queuer's concrete production impl).
|
||||
type autoscanQueuer = autoscan.Queuer
|
||||
|
||||
// RequestIntegrationLookup adapts the Requests repository to the autoscan
|
||||
// connection resolver's RequestIntegrationLookup: it resolves a soft-linked
|
||||
// Requests integration to its base URL and api key (the repo decrypts the key
|
||||
// on read, so this returns plaintext).
|
||||
type RequestIntegrationLookup struct {
|
||||
Repo *mediarequests.Repository
|
||||
}
|
||||
|
||||
func (l RequestIntegrationLookup) Get(ctx context.Context, integrationID string) (baseURL, apiKey string, err error) {
|
||||
integration, err := l.Repo.GetIntegration(ctx, integrationID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
// A reused Requests connection must honor the integration's live state. The
|
||||
// v1 poll gated on `WHERE ri.enabled = true`; here we surface a disabled or
|
||||
// unconfigured (blank base_url) integration as an error so the engine turns
|
||||
// it into a logged skip / RecordError rather than polling an unusable target.
|
||||
if err := checkRequestIntegrationUsable(integrationID, integration.Enabled, integration.BaseURL); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return integration.BaseURL, integration.APIKeyRef, nil
|
||||
}
|
||||
|
||||
// checkRequestIntegrationUsable returns a non-nil error when a linked Requests
|
||||
// integration cannot be polled: it is disabled, or it has no base_url. Extracted
|
||||
// as a pure function so the gating is unit-testable without a DB-backed repo.
|
||||
func checkRequestIntegrationUsable(integrationID string, enabled bool, baseURL string) error {
|
||||
if !enabled {
|
||||
return fmt.Errorf("linked requests integration %q is disabled", integrationID)
|
||||
}
|
||||
if strings.TrimSpace(baseURL) == "" {
|
||||
return fmt.Errorf("linked requests integration %q has no base_url configured", integrationID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PluginScanSourceAdapter adapts plugins.Service to autoscan.ScanSourceResolver.
|
||||
// plugins.Service.ScanSourceClient returns the concrete
|
||||
// *pluginhost.ScanSourceClient; that concrete type satisfies
|
||||
// autoscan.PollChangesClient (it has the matching PollChanges method), so the
|
||||
// adapter declares the exported interface as its return type and returns the
|
||||
// concrete value (Go has no return-type covariance, so the method signature must
|
||||
// name the interface exactly to satisfy ScanSourceResolver).
|
||||
type PluginScanSourceAdapter struct {
|
||||
Svc *plugins.Service
|
||||
}
|
||||
|
||||
func (a PluginScanSourceAdapter) ScanSourceClient(ctx context.Context, pluginID, capabilityID string) (autoscan.PollChangesClient, error) {
|
||||
return a.Svc.ScanSourceClientByPluginID(ctx, pluginID, capabilityID)
|
||||
}
|
||||
|
||||
// scanSourceCapabilityType is the plugin capability type autoscan discovery
|
||||
// enumerates.
|
||||
const scanSourceCapabilityType = "scan_source.v1"
|
||||
|
||||
// PluginScanSourceLister adapts the plugin installation store to
|
||||
// autoscan.ScanSourceLister: it enumerates every installed scan_source.v1
|
||||
// capability across ALL installed plugins, regardless of enabled state.
|
||||
type PluginScanSourceLister struct {
|
||||
Store *plugins.InstallationStore
|
||||
}
|
||||
|
||||
func (l PluginScanSourceLister) ListScanSources(ctx context.Context) ([]autoscan.DiscoveredSource, error) {
|
||||
installations, err := l.Store.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []autoscan.DiscoveredSource
|
||||
for _, inst := range installations {
|
||||
caps, err := l.Store.ListCapabilities(ctx, inst.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range caps {
|
||||
if c == nil || c.Type != scanSourceCapabilityType {
|
||||
continue
|
||||
}
|
||||
out = append(out, autoscan.DiscoveredSource{
|
||||
PluginID: inst.PluginID,
|
||||
CapabilityID: c.ID,
|
||||
DisplayName: scanSourceDisplayName(inst.PluginID, c),
|
||||
})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scanSourceDisplayName derives a human-friendly label for a scan_source
|
||||
// capability: the capability manifest's display_name when present, else the
|
||||
// plugin id (with the capability id appended when it adds information).
|
||||
func scanSourceDisplayName(pluginID string, c *plugins.Capability) string {
|
||||
if c != nil && c.Metadata != nil {
|
||||
if name, ok := c.Metadata["display_name"].(string); ok && strings.TrimSpace(name) != "" {
|
||||
return strings.TrimSpace(name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case pluginID != "" && c != nil && c.ID != "":
|
||||
return pluginID + " / " + c.ID
|
||||
case pluginID != "":
|
||||
return pluginID
|
||||
case c != nil:
|
||||
return c.ID
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// BuildAutoscanService wires the v2 autoscan engine from its concrete
|
||||
// dependencies. Both the HTTP router (manual trigger) and the background poll
|
||||
// task share this constructor so the adapter wiring lives in exactly one place.
|
||||
// Credentials are now decrypted inline by the autoscan/requests repos, so there
|
||||
// is no separate secret resolver to thread.
|
||||
func BuildAutoscanService(
|
||||
repo *autoscan.Repository,
|
||||
pluginService *plugins.Service,
|
||||
installationStore *plugins.InstallationStore,
|
||||
requestsRepo *mediarequests.Repository,
|
||||
folderRepo *catalog.FolderRepository,
|
||||
queue autoscanQueuer,
|
||||
redisClient *redis.Client,
|
||||
) *autoscan.Service {
|
||||
provider := autoscan.NewPluginProvider(PluginScanSourceAdapter{pluginService})
|
||||
connRes := autoscan.NewConnectionResolver(RequestIntegrationLookup{requestsRepo})
|
||||
svc := autoscan.NewService(
|
||||
repo,
|
||||
provider,
|
||||
connRes,
|
||||
scantrigger.NewResolver(folderRepo),
|
||||
queue,
|
||||
autoscan.NewRedisSuppressor(redisClient),
|
||||
PluginScanSourceLister{installationStore},
|
||||
)
|
||||
// Wire the connection-test + rewrite-suggester deps: a (long-timeout)
|
||||
// arr root-folder/status client and a Silo media-folder lister.
|
||||
svc.SetSuggesterDeps(
|
||||
autoscan.NewArrRootFolderClient(nil),
|
||||
autoscan.NewCatalogFolderLister(folderRepo),
|
||||
)
|
||||
return svc
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
mediarequests "github.com/Silo-Server/silo-server/internal/requests"
|
||||
)
|
||||
|
||||
// TestCheckRequestIntegrationUsable verifies the Fix 4 gate: a reused Requests
|
||||
// connection is rejected when the integration is disabled or has no base_url, so
|
||||
// the autoscan engine skips it instead of polling an unusable target.
|
||||
func TestCheckRequestIntegrationUsable(t *testing.T) {
|
||||
t.Run("enabled with base_url is usable", func(t *testing.T) {
|
||||
if err := checkRequestIntegrationUsable("req-1", true, "http://radarr:7878"); err != nil {
|
||||
t.Fatalf("expected usable, got %v", err)
|
||||
}
|
||||
})
|
||||
t.Run("disabled is rejected", func(t *testing.T) {
|
||||
if err := checkRequestIntegrationUsable("req-1", false, "http://radarr:7878"); err == nil {
|
||||
t.Fatal("expected error for disabled integration")
|
||||
}
|
||||
})
|
||||
t.Run("blank base_url is rejected", func(t *testing.T) {
|
||||
if err := checkRequestIntegrationUsable("req-1", true, " "); err == nil {
|
||||
t.Fatal("expected error for blank base_url")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestCheckRequestIntegrationUsableGatesReuse is the Task 17 regression test. It
|
||||
// locks the autoscan→requests credential-reuse behavior after the
|
||||
// request_integrations table was generalized into the two-tier connection
|
||||
// registry (Task 9/10): autoscan must keep reusing the Sonarr/Radarr connection
|
||||
// rows via the same enabled + base_url gate, regardless of the added
|
||||
// capability/installation columns.
|
||||
func TestCheckRequestIntegrationUsableGatesReuse(t *testing.T) {
|
||||
// Enabled + non-blank base_url -> usable (autoscan can reuse the connection).
|
||||
if err := checkRequestIntegrationUsable("int-1", true, "http://sonarr:8989"); err != nil {
|
||||
t.Fatalf("enabled integration with base_url should be usable, got: %v", err)
|
||||
}
|
||||
// Disabled -> error (autoscan must not poll a disabled connection).
|
||||
if err := checkRequestIntegrationUsable("int-1", false, "http://sonarr:8989"); err == nil {
|
||||
t.Fatal("disabled integration must be rejected")
|
||||
}
|
||||
// Blank base_url -> error.
|
||||
if err := checkRequestIntegrationUsable("int-1", true, " "); err == nil {
|
||||
t.Fatal("blank base_url must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
// Compile-time guard: autoscan's RequestIntegrationLookup reads these fields off
|
||||
// mediarequests.Integration. If any is renamed/removed, autoscan reuse breaks —
|
||||
// fail the build here rather than silently at runtime.
|
||||
var _ = func(in mediarequests.Integration) (string, string, bool) {
|
||||
return in.BaseURL, in.APIKeyRef, in.Enabled
|
||||
}
|
||||
+308
-91
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -20,14 +21,18 @@ import (
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/adminjob"
|
||||
"github.com/Silo-Server/silo-server/internal/ai/llm"
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/config"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
subtitleai "github.com/Silo-Server/silo-server/internal/subtitles/ai"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
@@ -112,20 +117,34 @@ func NewAdminHandler(
|
||||
|
||||
// createUserRequest represents the JSON body for POST /admin/users.
|
||||
type createUserRequest struct {
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions []string `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions createStringSliceField `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
}
|
||||
|
||||
type createStringSliceField struct {
|
||||
Set bool
|
||||
Value []string
|
||||
}
|
||||
|
||||
func (f *createStringSliceField) UnmarshalJSON(data []byte) error {
|
||||
f.Set = true
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
f.Value = []string{}
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(data, &f.Value)
|
||||
}
|
||||
|
||||
type updateLibraryIDsField struct {
|
||||
@@ -373,6 +392,9 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
req.Username = auth.NormalizeUsername(req.Username)
|
||||
req.Email = auth.NormalizeEmail(req.Email)
|
||||
|
||||
if req.Username == "" || req.Email == "" || req.Password == "" || req.Role == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Username, email, password, and role are required")
|
||||
return
|
||||
@@ -387,7 +409,11 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "max_profiles must be at least 1")
|
||||
return
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(req.Permissions)
|
||||
permissions := auth.DefaultUserPermissions()
|
||||
if req.Permissions.Set {
|
||||
permissions = req.Permissions.Value
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(permissions)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
@@ -460,7 +486,7 @@ func (h *AdminHandler) HandleUpdateUser(w http.ResponseWriter, r *http.Request)
|
||||
permissions = &normalized
|
||||
}
|
||||
|
||||
err = h.userRepo.Update(r.Context(), id, models.UpdateUserInput{
|
||||
updateInput := models.UpdateUserInput{
|
||||
Username: req.Username,
|
||||
Email: req.Email,
|
||||
Password: req.Password,
|
||||
@@ -474,18 +500,46 @@ func (h *AdminHandler) HandleUpdateUser(w http.ResponseWriter, r *http.Request)
|
||||
MaxProfiles: req.MaxProfiles,
|
||||
DownloadAllowed: req.DownloadAllowed,
|
||||
DownloadTranscodeAllowed: req.DownloadTranscodeAllowed,
|
||||
})
|
||||
}
|
||||
|
||||
var currentUser *models.User
|
||||
if updateMayRequireSessionRevocation(updateInput) {
|
||||
currentUser, err = h.userRepo.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
if auth.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "User not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch user")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err = h.userRepo.Update(r.Context(), id, updateInput)
|
||||
if err != nil {
|
||||
if auth.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "User not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update user")
|
||||
return
|
||||
}
|
||||
if updateRequiresSessionRevocation(req) {
|
||||
if updateRequiresSessionRevocation(currentUser, updateInput) {
|
||||
if err := h.revokeUserSessions(r.Context(), id); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to revoke updated user sessions")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Publish a domain event rather than creating the notification inline; the
|
||||
// notifications materializer's system matcher turns it into the user's
|
||||
// "Password changed" notice, keeping all notification creation event-driven.
|
||||
if req.Password != nil && h.RealtimeHub != nil {
|
||||
_ = h.RealtimeHub.EventsHub().PublishJSON(r.Context(),
|
||||
evt.ChannelSessions, notifications.EventUserPasswordChanged, nil,
|
||||
evt.PublishOptions{UserID: id})
|
||||
}
|
||||
|
||||
user, err := h.userRepo.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to fetch updated user")
|
||||
@@ -747,13 +801,35 @@ func (h *AdminHandler) HandleListUserProfiles(w http.ResponseWriter, r *http.Req
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func updateRequiresSessionRevocation(req updateUserRequest) bool {
|
||||
return req.Password != nil ||
|
||||
req.Role != nil ||
|
||||
req.Enabled != nil ||
|
||||
req.LibraryIDs.Set ||
|
||||
req.Permissions.Set ||
|
||||
req.MaxPlaybackQuality != nil
|
||||
func updateMayRequireSessionRevocation(input models.UpdateUserInput) bool {
|
||||
return input.Password != nil ||
|
||||
input.Role != nil ||
|
||||
input.Enabled != nil ||
|
||||
input.Permissions != nil ||
|
||||
input.MaxPlaybackQuality != nil
|
||||
}
|
||||
|
||||
func updateRequiresSessionRevocation(current *models.User, input models.UpdateUserInput) bool {
|
||||
if input.Password != nil {
|
||||
return true
|
||||
}
|
||||
if current == nil {
|
||||
return updateMayRequireSessionRevocation(input)
|
||||
}
|
||||
if input.Role != nil && *input.Role != current.Role {
|
||||
return true
|
||||
}
|
||||
if input.Enabled != nil && *input.Enabled != current.Enabled {
|
||||
return true
|
||||
}
|
||||
if input.Permissions != nil && !slices.Equal(*input.Permissions, current.Permissions) {
|
||||
return true
|
||||
}
|
||||
if input.MaxPlaybackQuality != nil &&
|
||||
access.NormalizePlaybackQuality(*input.MaxPlaybackQuality) != access.NormalizePlaybackQuality(current.MaxPlaybackQuality) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *AdminHandler) revokeUserSessions(ctx context.Context, userID int) error {
|
||||
@@ -830,6 +906,9 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
var resp AdminStats
|
||||
|
||||
if h.StatsSource != nil {
|
||||
if isTruthyQuery(r.URL.Query().Get("refresh")) {
|
||||
h.StatsSource.Invalidate()
|
||||
}
|
||||
stats, err := h.StatsSource.Get(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to get stats")
|
||||
@@ -856,6 +935,15 @@ func (h *AdminHandler) HandleGetStats(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func isTruthyQuery(value string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AdminHandler) invalidateStats(ctx context.Context, channel, eventType, payload string) {
|
||||
if h.StatsSource != nil {
|
||||
h.StatsSource.Invalidate()
|
||||
@@ -966,6 +1054,7 @@ type UpdateItemMetadataRequest struct {
|
||||
FirstAirDate *string `json:"first_air_date"`
|
||||
LastAirDate *string `json:"last_air_date"`
|
||||
AirTime *string `json:"air_time"`
|
||||
AirTimezone *string `json:"air_timezone"`
|
||||
AirDate *string `json:"air_date"`
|
||||
Status *string `json:"status"`
|
||||
RatingIMDB *float64 `json:"rating_imdb"`
|
||||
@@ -993,6 +1082,14 @@ func (h *AdminHandler) HandleUpdateItemMetadata(w http.ResponseWriter, r *http.R
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if req.AirTimezone != nil {
|
||||
trimmed := strings.TrimSpace(*req.AirTimezone)
|
||||
req.AirTimezone = &trimmed
|
||||
if !catalog.ValidateAirTimezone(trimmed) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "air_timezone must be a valid IANA timezone")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
upd := catalog.MetadataUpdate{
|
||||
Title: req.Title, SortTitle: req.SortTitle, OriginalTitle: req.OriginalTitle,
|
||||
@@ -1000,7 +1097,7 @@ func (h *AdminHandler) HandleUpdateItemMetadata(w http.ResponseWriter, r *http.R
|
||||
Year: req.Year, Runtime: req.Runtime,
|
||||
Genres: req.Genres, Studios: req.Studios, Networks: req.Networks, Countries: req.Countries,
|
||||
ReleaseDate: req.ReleaseDate, FirstAirDate: req.FirstAirDate, LastAirDate: req.LastAirDate,
|
||||
AirTime: req.AirTime,
|
||||
AirTime: req.AirTime, AirTimezone: req.AirTimezone,
|
||||
AirDate: req.AirDate, Status: req.Status,
|
||||
RatingIMDB: req.RatingIMDB, RatingTMDB: req.RatingTMDB,
|
||||
RatingRTCritic: req.RatingRTCritic, RatingRTAudience: req.RatingRTAudience,
|
||||
@@ -1053,28 +1150,10 @@ func (h *AdminHandler) HandleUpdateItemMetadata(w http.ResponseWriter, r *http.R
|
||||
|
||||
// --- Server Settings endpoints ---
|
||||
|
||||
var sensitiveSettingKeys = map[string]bool{
|
||||
"auth.jwt_secret": true,
|
||||
"s3.public_access_key": true,
|
||||
"s3.public_secret_key": true,
|
||||
"s3.public_token_secret": true,
|
||||
"s3.private_access_key": true,
|
||||
"s3.private_secret_key": true,
|
||||
"s3.user_db_access_key": true,
|
||||
"s3.user_db_secret_key": true,
|
||||
"redis.url": true,
|
||||
"recommendations.openai_api_key": true,
|
||||
"recommendations.embedding_auth_token": true,
|
||||
"tmdb.api_key": true,
|
||||
"introdb.api_key": true,
|
||||
"mdblist.api_key": true,
|
||||
"requests.radarr.api_key": true,
|
||||
"requests.sonarr.api_key": true,
|
||||
"watchsync.trakt.client_id": true,
|
||||
"watchsync.trakt.client_secret": true,
|
||||
"watchsync.simkl.client_id": true,
|
||||
"watchsync.simkl.client_secret": true,
|
||||
}
|
||||
// sensitiveSettingKeys is the audited allowlist of secret-bearing settings
|
||||
// keys, shared with the at-rest encryption decorator so redaction and
|
||||
// encryption can never drift apart. See catalog.SensitiveSettingKeys.
|
||||
var sensitiveSettingKeys = catalog.SensitiveSettingKeys
|
||||
|
||||
// HandleGetSettings handles GET /admin/settings.
|
||||
func (h *AdminHandler) HandleGetSettings(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1149,6 +1228,9 @@ func (h *AdminHandler) HandleGetSensitiveStatus(w http.ResponseWriter, r *http.R
|
||||
type adminSettingResponse struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
// RestartRequired reports whether the saved value only takes effect
|
||||
// after a server restart (set on update responses only).
|
||||
RestartRequired bool `json:"restart_required,omitempty"`
|
||||
}
|
||||
|
||||
type adminSettingsListResponse struct {
|
||||
@@ -1202,6 +1284,9 @@ type adminDeviceDetailResponse struct {
|
||||
DeviceID string `json:"device_id"`
|
||||
DeviceName string `json:"device_name"`
|
||||
DevicePlatform string `json:"device_platform"`
|
||||
OverrideCount int `json:"override_count"`
|
||||
ProfileCount int `json:"profile_count"`
|
||||
Profiles []adminDeviceProfileSummary `json:"profiles"`
|
||||
LastUpdated string `json:"last_updated"`
|
||||
Settings []adminDeviceSettingResponse `json:"settings"`
|
||||
}
|
||||
@@ -1444,6 +1529,12 @@ func (h *AdminHandler) HandleUpdateUserDeviceSetting(w http.ResponseWriter, r *h
|
||||
if existing != nil {
|
||||
entry.DeviceName = existing.DeviceName
|
||||
entry.DevicePlatform = existing.DevicePlatform
|
||||
} else if registered, err := registeredDeviceForProfile(r.Context(), store, profileID, deviceID); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
|
||||
return
|
||||
} else if registered != nil {
|
||||
entry.DeviceName = registered.DeviceName
|
||||
entry.DevicePlatform = registered.DevicePlatform
|
||||
}
|
||||
if err := store.SetDeviceSetting(r.Context(), entry); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update device setting")
|
||||
@@ -1566,6 +1657,10 @@ func (h *AdminHandler) HandleListDevices(w http.ResponseWriter, r *http.Request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list device settings: %w", err)
|
||||
}
|
||||
devices, err := listRegisteredDevices(gctx, store)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list devices: %w", err)
|
||||
}
|
||||
profileNames, err := listProfileNamesByID(gctx, store)
|
||||
if err != nil {
|
||||
slog.Warn("admin list devices profile lookup failed",
|
||||
@@ -1574,7 +1669,14 @@ func (h *AdminHandler) HandleListDevices(w http.ResponseWriter, r *http.Request)
|
||||
)
|
||||
profileNames = map[string]string{}
|
||||
}
|
||||
perUser[i] = buildAdminDeviceSummaries(user.ID, user.Username, user.Email, entries, profileNames)
|
||||
perUser[i] = buildAdminDeviceSummaries(
|
||||
user.ID,
|
||||
user.Username,
|
||||
user.Email,
|
||||
entries,
|
||||
devices,
|
||||
profileNames,
|
||||
)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -1638,6 +1740,11 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
|
||||
return
|
||||
}
|
||||
registeredDevices, err := listRegisteredDevices(r.Context(), store)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load device")
|
||||
return
|
||||
}
|
||||
profileNames, err := listProfileNamesByID(r.Context(), store)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list profiles")
|
||||
@@ -1650,12 +1757,26 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
|
||||
deviceEntries = append(deviceEntries, entry)
|
||||
}
|
||||
}
|
||||
if len(deviceEntries) == 0 {
|
||||
deviceRegistrations := make([]userstore.DeviceEntry, 0)
|
||||
for _, entry := range registeredDevices {
|
||||
if entry.DeviceID == deviceID {
|
||||
deviceRegistrations = append(deviceRegistrations, entry)
|
||||
}
|
||||
}
|
||||
summaries := buildAdminDeviceSummaries(
|
||||
user.ID,
|
||||
user.Username,
|
||||
user.Email,
|
||||
deviceEntries,
|
||||
deviceRegistrations,
|
||||
profileNames,
|
||||
)
|
||||
if len(summaries) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Device not found")
|
||||
return
|
||||
}
|
||||
|
||||
summary := buildAdminDeviceSummaries(user.ID, user.Username, user.Email, deviceEntries, profileNames)[0]
|
||||
summary := summaries[0]
|
||||
writeJSON(w, http.StatusOK, adminDeviceDetailResponse{
|
||||
UserID: user.ID,
|
||||
Username: user.Username,
|
||||
@@ -1663,11 +1784,41 @@ func (h *AdminHandler) HandleGetDevice(w http.ResponseWriter, r *http.Request) {
|
||||
DeviceID: summary.DeviceID,
|
||||
DeviceName: summary.DeviceName,
|
||||
DevicePlatform: summary.DevicePlatform,
|
||||
OverrideCount: summary.OverrideCount,
|
||||
ProfileCount: summary.ProfileCount,
|
||||
Profiles: summary.Profiles,
|
||||
LastUpdated: summary.LastUpdated,
|
||||
Settings: buildAdminDeviceSettingsResponse(user.ID, profileNames, deviceEntries).Settings,
|
||||
})
|
||||
}
|
||||
|
||||
func listRegisteredDevices(ctx context.Context, store userstore.UserStore) ([]userstore.DeviceEntry, error) {
|
||||
registry, ok := store.(userstore.DeviceRegistry)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return registry.ListDevices(ctx)
|
||||
}
|
||||
|
||||
func registeredDeviceForProfile(
|
||||
ctx context.Context,
|
||||
store userstore.UserStore,
|
||||
profileID string,
|
||||
deviceID string,
|
||||
) (*userstore.DeviceEntry, error) {
|
||||
devices, err := listRegisteredDevices(ctx, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, device := range devices {
|
||||
if device.ProfileID == profileID && device.DeviceID == deviceID {
|
||||
matched := device
|
||||
return &matched, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func buildAdminDeviceSettingsResponse(userID int, profileNames map[string]string, entries []userstore.DeviceSettingEntry) adminDeviceSettingsListResponse {
|
||||
resp := adminDeviceSettingsListResponse{
|
||||
Settings: make([]adminDeviceSettingResponse, 0, len(entries)),
|
||||
@@ -1693,6 +1844,7 @@ func buildAdminDeviceSummaries(
|
||||
username string,
|
||||
email string,
|
||||
entries []userstore.DeviceSettingEntry,
|
||||
registeredDevices []userstore.DeviceEntry,
|
||||
profileNames map[string]string,
|
||||
) []adminDeviceSummaryResponse {
|
||||
type profileAccumulator struct {
|
||||
@@ -1706,10 +1858,10 @@ func buildAdminDeviceSummaries(
|
||||
}
|
||||
|
||||
byDevice := make(map[string]*summary)
|
||||
for _, entry := range entries {
|
||||
deviceID := strings.TrimSpace(entry.DeviceID)
|
||||
|
||||
ensureDevice := func(deviceID, deviceName, devicePlatform, lastUpdated string) *summary {
|
||||
if deviceID == "" {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
current, ok := byDevice[deviceID]
|
||||
if !ok {
|
||||
@@ -1719,49 +1871,86 @@ func buildAdminDeviceSummaries(
|
||||
Username: username,
|
||||
Email: email,
|
||||
DeviceID: deviceID,
|
||||
DeviceName: entry.DeviceName,
|
||||
DevicePlatform: entry.DevicePlatform,
|
||||
LastUpdated: entry.UpdatedAt,
|
||||
DeviceName: deviceName,
|
||||
DevicePlatform: devicePlatform,
|
||||
LastUpdated: lastUpdated,
|
||||
},
|
||||
keys: make(map[string]struct{}),
|
||||
profiles: make(map[string]*profileAccumulator),
|
||||
}
|
||||
byDevice[deviceID] = current
|
||||
}
|
||||
current.keys[entry.ProfileID+":"+entry.Key] = struct{}{}
|
||||
if current.device.DeviceName == "" && deviceName != "" {
|
||||
current.device.DeviceName = deviceName
|
||||
}
|
||||
if current.device.DevicePlatform == "" && devicePlatform != "" {
|
||||
current.device.DevicePlatform = devicePlatform
|
||||
}
|
||||
if lastUpdated > current.device.LastUpdated {
|
||||
current.device.LastUpdated = lastUpdated
|
||||
if deviceName != "" {
|
||||
current.device.DeviceName = deviceName
|
||||
}
|
||||
if devicePlatform != "" {
|
||||
current.device.DevicePlatform = devicePlatform
|
||||
}
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
ensureProfile := func(current *summary, profileID, lastUpdated string) *profileAccumulator {
|
||||
if current == nil || profileID == "" {
|
||||
return nil
|
||||
}
|
||||
profile, exists := current.profiles[profileID]
|
||||
if !exists {
|
||||
profile = &profileAccumulator{
|
||||
summary: adminDeviceProfileSummary{
|
||||
ProfileID: profileID,
|
||||
ProfileName: profileNames[profileID],
|
||||
LastUpdated: lastUpdated,
|
||||
},
|
||||
keys: make(map[string]struct{}),
|
||||
}
|
||||
current.profiles[profileID] = profile
|
||||
return profile
|
||||
}
|
||||
if lastUpdated > profile.summary.LastUpdated {
|
||||
profile.summary.LastUpdated = lastUpdated
|
||||
}
|
||||
return profile
|
||||
}
|
||||
|
||||
for _, device := range registeredDevices {
|
||||
deviceID := strings.TrimSpace(device.DeviceID)
|
||||
profileID := strings.TrimSpace(device.ProfileID)
|
||||
current := ensureDevice(
|
||||
deviceID,
|
||||
device.DeviceName,
|
||||
device.DevicePlatform,
|
||||
device.LastSeenAt,
|
||||
)
|
||||
ensureProfile(current, profileID, device.LastSeenAt)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
deviceID := strings.TrimSpace(entry.DeviceID)
|
||||
profileID := strings.TrimSpace(entry.ProfileID)
|
||||
if profileID != "" {
|
||||
profile, exists := current.profiles[profileID]
|
||||
if !exists {
|
||||
profile = &profileAccumulator{
|
||||
summary: adminDeviceProfileSummary{
|
||||
ProfileID: profileID,
|
||||
ProfileName: profileNames[profileID],
|
||||
LastUpdated: entry.UpdatedAt,
|
||||
},
|
||||
keys: make(map[string]struct{}),
|
||||
}
|
||||
current.profiles[profileID] = profile
|
||||
}
|
||||
current := ensureDevice(
|
||||
deviceID,
|
||||
entry.DeviceName,
|
||||
entry.DevicePlatform,
|
||||
entry.UpdatedAt,
|
||||
)
|
||||
if current == nil {
|
||||
continue
|
||||
}
|
||||
if profileID != "" && entry.Key != "" {
|
||||
current.keys[profileID+":"+entry.Key] = struct{}{}
|
||||
}
|
||||
profile := ensureProfile(current, profileID, entry.UpdatedAt)
|
||||
if profile != nil && entry.Key != "" {
|
||||
profile.keys[entry.Key] = struct{}{}
|
||||
if entry.UpdatedAt > profile.summary.LastUpdated {
|
||||
profile.summary.LastUpdated = entry.UpdatedAt
|
||||
}
|
||||
}
|
||||
if current.device.DeviceName == "" && entry.DeviceName != "" {
|
||||
current.device.DeviceName = entry.DeviceName
|
||||
}
|
||||
if current.device.DevicePlatform == "" && entry.DevicePlatform != "" {
|
||||
current.device.DevicePlatform = entry.DevicePlatform
|
||||
}
|
||||
if entry.UpdatedAt > current.device.LastUpdated {
|
||||
current.device.LastUpdated = entry.UpdatedAt
|
||||
if entry.DeviceName != "" {
|
||||
current.device.DeviceName = entry.DeviceName
|
||||
}
|
||||
if entry.DevicePlatform != "" {
|
||||
current.device.DevicePlatform = entry.DevicePlatform
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1928,6 +2117,33 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
} else {
|
||||
req.Value = normalized
|
||||
}
|
||||
case "ai.asr_base_url":
|
||||
if llm.IsChatOnlyGateway(req.Value) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request",
|
||||
"This endpoint cannot produce timestamped transcriptions (chat-only gateway). "+
|
||||
"Use a self-hosted Whisper server (faster-whisper/speaches), api.groq.com/openai, or api.openai.com.")
|
||||
return
|
||||
}
|
||||
case "metadata_ai.on_view":
|
||||
switch req.Value {
|
||||
case "off", "button", "auto":
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, "bad_request",
|
||||
"metadata_ai.on_view must be off, button, or auto")
|
||||
return
|
||||
}
|
||||
case "subtitle_ai.transcribe_quota_jobs":
|
||||
if n, err := strconv.Atoi(req.Value); err != nil || n < 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request",
|
||||
"subtitle_ai.transcribe_quota_jobs must be an integer >= 0 (0 = unlimited)")
|
||||
return
|
||||
}
|
||||
case "subtitle_ai.transcribe_quota_period":
|
||||
if !subtitleai.ValidQuotaPeriod(req.Value) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request",
|
||||
"subtitle_ai.transcribe_quota_period must be day, week, or month")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := h.SettingsRepo.Set(r.Context(), key, req.Value); err != nil {
|
||||
@@ -1943,9 +2159,10 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
h.OnServerSettingUpdated(r.Context(), key, req.Value)
|
||||
}
|
||||
|
||||
restartRequired := config.RestartRequired(key)
|
||||
if sensitiveSettingKeys[key] {
|
||||
writeJSON(w, http.StatusOK, adminSettingResponse{Key: key})
|
||||
writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, RestartRequired: restartRequired})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: req.Value})
|
||||
writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: req.Value, RestartRequired: restartRequired})
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
)
|
||||
|
||||
const adminJobDownloadExpiry = 15 * time.Minute
|
||||
@@ -23,12 +24,22 @@ type AdminJobArtifactStore interface {
|
||||
PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error)
|
||||
}
|
||||
|
||||
type AdminJobsHandler struct {
|
||||
repo *adminjob.Repository
|
||||
store AdminJobArtifactStore
|
||||
type adminJobRepository interface {
|
||||
List(ctx context.Context, opts adminjob.ListJobsOptions) ([]*models.AdminJob, error)
|
||||
GetByID(ctx context.Context, id string) (*models.AdminJob, error)
|
||||
Cancel(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error)
|
||||
CancelQueued(ctx context.Context, id, message string, expiresAt time.Time) (*models.AdminJob, error)
|
||||
UpdateProgress(ctx context.Context, id string, current, total int, message string) error
|
||||
}
|
||||
|
||||
func NewAdminJobsHandler(repo *adminjob.Repository, store AdminJobArtifactStore) *AdminJobsHandler {
|
||||
type AdminJobsHandler struct {
|
||||
repo adminJobRepository
|
||||
store AdminJobArtifactStore
|
||||
CancelRegistry *adminjob.CancelRegistry
|
||||
RealtimeHub *notifications.Hub
|
||||
}
|
||||
|
||||
func NewAdminJobsHandler(repo adminJobRepository, store AdminJobArtifactStore) *AdminJobsHandler {
|
||||
return &AdminJobsHandler{repo: repo, store: store}
|
||||
}
|
||||
|
||||
@@ -120,6 +131,105 @@ func (h *AdminJobsHandler) HandleGet(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, adminJobToResponseForClaims(r, job, h.store, claims))
|
||||
}
|
||||
|
||||
func (h *AdminJobsHandler) HandleCancel(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.repo == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "not_configured", "Admin jobs are not configured")
|
||||
return
|
||||
}
|
||||
|
||||
id := chi.URLParam(r, "id")
|
||||
if id == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Job ID is required")
|
||||
return
|
||||
}
|
||||
|
||||
job, err := h.repo.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
if errors.Is(err, adminjob.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
|
||||
return
|
||||
}
|
||||
|
||||
if job.JobType != adminjob.JobTypeLibraryRefresh {
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Only library metadata refresh jobs can be cancelled")
|
||||
return
|
||||
}
|
||||
|
||||
switch job.Status {
|
||||
case adminjob.StatusQueued:
|
||||
cancelled, cancelErr := h.repo.CancelQueued(
|
||||
r.Context(),
|
||||
id,
|
||||
"Library metadata refresh cancelled",
|
||||
time.Now().UTC().Add(7*24*time.Hour),
|
||||
)
|
||||
if cancelErr != nil {
|
||||
if errors.Is(cancelErr, adminjob.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(cancelErr, adminjob.ErrJobNotCancellable) {
|
||||
advanced, loadErr := h.repo.GetByID(r.Context(), id)
|
||||
if loadErr != nil {
|
||||
if errors.Is(loadErr, adminjob.ErrJobNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Job not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
|
||||
return
|
||||
}
|
||||
if advanced.Status == adminjob.StatusRunning {
|
||||
h.requestRunningCancellation(w, r, advanced)
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Job is no longer cancellable")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel job")
|
||||
return
|
||||
}
|
||||
if h.RealtimeHub != nil {
|
||||
publishEventJob(
|
||||
r.Context(),
|
||||
h.RealtimeHub.EventsHub(),
|
||||
string(notifications.TypeJobCancelled),
|
||||
cancelled,
|
||||
)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, adminJobToResponse(r, cancelled, h.store))
|
||||
case adminjob.StatusRunning:
|
||||
h.requestRunningCancellation(w, r, job)
|
||||
default:
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Job is no longer cancellable")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AdminJobsHandler) requestRunningCancellation(w http.ResponseWriter, r *http.Request, job *models.AdminJob) {
|
||||
if h.CancelRegistry == nil || !h.CancelRegistry.Cancel(job.ID) {
|
||||
writeError(w, http.StatusConflict, "not_cancellable", "Running job is not cancellable on this node")
|
||||
return
|
||||
}
|
||||
if err := h.repo.UpdateProgress(
|
||||
r.Context(),
|
||||
job.ID,
|
||||
job.ProgressCurrent,
|
||||
job.ProgressTotal,
|
||||
"Cancellation requested",
|
||||
); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to request cancellation")
|
||||
return
|
||||
}
|
||||
updated, err := h.repo.GetByID(r.Context(), job.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load admin job")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, adminJobToResponse(r, updated, h.store))
|
||||
}
|
||||
|
||||
func adminJobToResponse(r *http.Request, job *models.AdminJob, store AdminJobArtifactStore) adminJobResponse {
|
||||
resp := adminJobResponse{
|
||||
ID: job.ID,
|
||||
|
||||
@@ -2,12 +2,21 @@ package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/adminjob"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
)
|
||||
|
||||
func TestCanReadAdminJob_AdminCanReadAnyJob(t *testing.T) {
|
||||
@@ -79,3 +88,195 @@ func TestAdminJobToResponseForClaims_NonAdminSanitizesItemRefreshPayloads(t *tes
|
||||
t.Fatalf("ErrorMessage = %q, want stripped for non-admin", resp.ErrorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminJobsHandleCancel_RunningRegistryMissDoesNotUpdateProgress(t *testing.T) {
|
||||
repo := &fakeAdminJobRepository{
|
||||
job: &models.AdminJob{
|
||||
ID: "job-1",
|
||||
JobType: adminjob.JobTypeLibraryRefresh,
|
||||
Status: adminjob.StatusRunning,
|
||||
CreatedByUserID: 1,
|
||||
Message: "Running",
|
||||
RequestedAt: time.Now().UTC(),
|
||||
},
|
||||
}
|
||||
handler := NewAdminJobsHandler(repo, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
|
||||
|
||||
if rec.Code != http.StatusConflict {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusConflict)
|
||||
}
|
||||
if repo.updateProgressCalls != 0 {
|
||||
t.Fatalf("UpdateProgress called %d times, want 0", repo.updateProgressCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminJobsHandleCancel_QueuedPublishesCancelledEvent(t *testing.T) {
|
||||
repo := &fakeAdminJobRepository{
|
||||
job: &models.AdminJob{
|
||||
ID: "job-1",
|
||||
JobType: adminjob.JobTypeLibraryRefresh,
|
||||
Status: adminjob.StatusQueued,
|
||||
CreatedByUserID: 1,
|
||||
Message: "Queued",
|
||||
RequestedAt: time.Now().UTC(),
|
||||
},
|
||||
}
|
||||
hub := notifications.NewHub("test", &cache.NoopEventBus{})
|
||||
events, unsubscribe := hub.Subscribe()
|
||||
defer unsubscribe()
|
||||
|
||||
handler := NewAdminJobsHandler(repo, nil)
|
||||
handler.RealtimeHub = hub
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case event := <-events:
|
||||
if event.Type != notifications.TypeJobCancelled {
|
||||
t.Fatalf("event.Type = %q, want %q", event.Type, notifications.TypeJobCancelled)
|
||||
}
|
||||
if event.Job == nil || event.Job.ID != "job-1" {
|
||||
t.Fatalf("event.Job = %#v, want job-1", event.Job)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for job.cancelled event")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminJobsHandleCancel_QueuedSnapshotFallsBackToRunningCancellation(t *testing.T) {
|
||||
repo := &fakeAdminJobRepository{
|
||||
job: &models.AdminJob{
|
||||
ID: "job-1",
|
||||
JobType: adminjob.JobTypeLibraryRefresh,
|
||||
Status: adminjob.StatusQueued,
|
||||
CreatedByUserID: 1,
|
||||
Message: "Queued",
|
||||
RequestedAt: time.Now().UTC(),
|
||||
ProgressCurrent: 3,
|
||||
ProgressTotal: 10,
|
||||
},
|
||||
promoteToRunningOnCancelQueued: true,
|
||||
}
|
||||
cancelled := false
|
||||
registry := adminjob.NewCancelRegistry()
|
||||
unregister := registry.Register("job-1", func() {
|
||||
cancelled = true
|
||||
})
|
||||
defer unregister()
|
||||
|
||||
handler := NewAdminJobsHandler(repo, nil)
|
||||
handler.CancelRegistry = registry
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleCancel(rec, adminJobCancelRequest("job-1"))
|
||||
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusAccepted, rec.Body.String())
|
||||
}
|
||||
if !cancelled {
|
||||
t.Fatal("expected running job cancel registry to be invoked")
|
||||
}
|
||||
if repo.cancelQueuedCalls != 1 {
|
||||
t.Fatalf("CancelQueued called %d times, want 1", repo.cancelQueuedCalls)
|
||||
}
|
||||
if repo.cancelCalls != 0 {
|
||||
t.Fatalf("Cancel called %d times, want 0", repo.cancelCalls)
|
||||
}
|
||||
if repo.updateProgressCalls != 1 {
|
||||
t.Fatalf("UpdateProgress called %d times, want 1", repo.updateProgressCalls)
|
||||
}
|
||||
if repo.job.Status != adminjob.StatusRunning {
|
||||
t.Fatalf("job status = %q, want %q", repo.job.Status, adminjob.StatusRunning)
|
||||
}
|
||||
if repo.job.Message != "Cancellation requested" {
|
||||
t.Fatalf("job message = %q, want cancellation request", repo.job.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func adminJobCancelRequest(id string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/jobs/"+id+"/cancel", nil)
|
||||
routeCtx := chi.NewRouteContext()
|
||||
routeCtx.URLParams.Add("id", id)
|
||||
return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
|
||||
}
|
||||
|
||||
type fakeAdminJobRepository struct {
|
||||
job *models.AdminJob
|
||||
updateProgressCalls int
|
||||
cancelCalls int
|
||||
cancelQueuedCalls int
|
||||
promoteToRunningOnCancelQueued bool
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) List(context.Context, adminjob.ListJobsOptions) ([]*models.AdminJob, error) {
|
||||
if r.job == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return []*models.AdminJob{r.job}, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) GetByID(_ context.Context, id string) (*models.AdminJob, error) {
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return nil, adminjob.ErrJobNotFound
|
||||
}
|
||||
cp := *r.job
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) Cancel(_ context.Context, id, message string, _ time.Time) (*models.AdminJob, error) {
|
||||
r.cancelCalls++
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return nil, adminjob.ErrJobNotFound
|
||||
}
|
||||
if r.job.Status != adminjob.StatusQueued && r.job.Status != adminjob.StatusRunning {
|
||||
return nil, adminjob.ErrJobNotCancellable
|
||||
}
|
||||
cp := *r.job
|
||||
cp.Status = adminjob.StatusCancelled
|
||||
cp.Message = message
|
||||
r.job = &cp
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) CancelQueued(_ context.Context, id, message string, _ time.Time) (*models.AdminJob, error) {
|
||||
r.cancelQueuedCalls++
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return nil, adminjob.ErrJobNotFound
|
||||
}
|
||||
if r.promoteToRunningOnCancelQueued {
|
||||
cp := *r.job
|
||||
cp.Status = adminjob.StatusRunning
|
||||
r.job = &cp
|
||||
return nil, adminjob.ErrJobNotCancellable
|
||||
}
|
||||
if r.job.Status != adminjob.StatusQueued {
|
||||
return nil, adminjob.ErrJobNotCancellable
|
||||
}
|
||||
cp := *r.job
|
||||
cp.Status = adminjob.StatusCancelled
|
||||
cp.Message = message
|
||||
r.job = &cp
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (r *fakeAdminJobRepository) UpdateProgress(_ context.Context, id string, current, total int, message string) error {
|
||||
r.updateProgressCalls++
|
||||
if r.job == nil || r.job.ID != id {
|
||||
return adminjob.ErrJobNotFound
|
||||
}
|
||||
if message == "" {
|
||||
return errors.New("message is required")
|
||||
}
|
||||
r.job.ProgressCurrent = current
|
||||
r.job.ProgressTotal = total
|
||||
r.job.Message = message
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
// AdminMarkerProvidersHandler serves the per-provider marker config + key
|
||||
// validation API under the RequireAdmin group.
|
||||
type AdminMarkerProvidersHandler struct {
|
||||
Registry *markers.Registry
|
||||
Config *markers.ProviderConfigStore
|
||||
EventBus cache.EventBus
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewAdminMarkerProvidersHandler constructs the handler.
|
||||
func NewAdminMarkerProvidersHandler(registry *markers.Registry, config *markers.ProviderConfigStore, eventBus cache.EventBus, logger *slog.Logger) *AdminMarkerProvidersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &AdminMarkerProvidersHandler{Registry: registry, Config: config, EventBus: eventBus, logger: logger}
|
||||
}
|
||||
|
||||
type providerConfigResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
DisplayName string `json:"display_name,omitempty"`
|
||||
SourceType string `json:"source_type,omitempty"`
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
PluginInstallationID int `json:"plugin_installation_id,omitempty"`
|
||||
CapabilityID string `json:"capability_id,omitempty"`
|
||||
IsSubmitter bool `json:"is_submitter"`
|
||||
FetchEnabled bool `json:"fetch_enabled"`
|
||||
FetchPriority int `json:"fetch_priority"`
|
||||
ContributeEnabled bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
|
||||
type markerUserStatsResponse struct {
|
||||
Total int `json:"total"`
|
||||
Accepted int `json:"accepted"`
|
||||
Pending int `json:"pending"`
|
||||
Rejected int `json:"rejected"`
|
||||
AcceptanceRate float64 `json:"acceptance_rate"`
|
||||
CurrentStreak int `json:"current_streak"`
|
||||
BestStreak int `json:"best_streak"`
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) submitterIDs() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if _, ok := p.(markers.Submitter); ok {
|
||||
out[p.ID()] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) providerDescriptions() map[string]markers.ProviderDescriptor {
|
||||
out := map[string]markers.ProviderDescriptor{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
desc := markers.ProviderDescriptor{ID: p.ID()}
|
||||
if described, ok := p.(markers.DescribedProvider); ok {
|
||||
desc = described.ProviderDescription()
|
||||
}
|
||||
if desc.ID == "" {
|
||||
desc.ID = p.ID()
|
||||
}
|
||||
out[p.ID()] = desc
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HandleListProviders lists registered providers with their config + capability.
|
||||
func (h *AdminMarkerProvidersHandler) HandleListProviders(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
submitters := h.submitterIDs()
|
||||
descriptions := h.providerDescriptions()
|
||||
out := []providerConfigResponse{}
|
||||
if h.Registry != nil {
|
||||
for _, provider := range h.Registry.Providers() {
|
||||
c, ok := h.Config.Get(provider.ID())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
} else {
|
||||
for _, c := range h.Config.List() {
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FetchPriority != out[j].FetchPriority {
|
||||
return out[i].FetchPriority < out[j].FetchPriority
|
||||
}
|
||||
return out[i].Provider < out[j].Provider
|
||||
})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"providers": out})
|
||||
}
|
||||
|
||||
// HandleUpdateProvider updates a provider's config row.
|
||||
func (h *AdminMarkerProvidersHandler) HandleUpdateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider, err := decodedURLParam(r, "provider")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid provider ID")
|
||||
return
|
||||
}
|
||||
existing, ok := h.Config.Get(provider)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Unknown marker provider")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
FetchEnabled *bool `json:"fetch_enabled"`
|
||||
FetchPriority *int `json:"fetch_priority"`
|
||||
ContributeEnabled *bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal *bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence *float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if body.FetchEnabled != nil {
|
||||
existing.FetchEnabled = *body.FetchEnabled
|
||||
}
|
||||
if body.FetchPriority != nil {
|
||||
existing.FetchPriority = *body.FetchPriority
|
||||
}
|
||||
if body.ContributeEnabled != nil {
|
||||
existing.ContributeEnabled = *body.ContributeEnabled
|
||||
}
|
||||
if body.ContributeAutoLocal != nil {
|
||||
existing.ContributeAutoLocal = *body.ContributeAutoLocal
|
||||
}
|
||||
if body.ContributeMinConfidence != nil {
|
||||
v := *body.ContributeMinConfidence
|
||||
if v < 0 || v > 1 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "contribute_min_confidence must be between 0 and 1")
|
||||
return
|
||||
}
|
||||
existing.ContributeMinConfidence = v
|
||||
}
|
||||
if err := h.Config.Update(r.Context(), existing); err != nil {
|
||||
h.logger.Error("admin markers: update provider config failed", "provider", provider, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update provider")
|
||||
return
|
||||
}
|
||||
if h.EventBus != nil {
|
||||
_ = h.EventBus.Publish(r.Context(), cache.ChannelAdmin, cache.Event{
|
||||
Type: cache.EventMarkerProviderConfigChanged,
|
||||
Payload: provider,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, toProviderConfigResponse(existing, h.submitterIDs()[provider], h.providerDescriptions()[provider]))
|
||||
}
|
||||
|
||||
// HandleValidateProvider validates the provider's configured key and returns stats.
|
||||
func (h *AdminMarkerProvidersHandler) HandleValidateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Registry == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider, err := decodedURLParam(r, "provider")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid provider ID")
|
||||
return
|
||||
}
|
||||
var submitter markers.Submitter
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if p.ID() == provider {
|
||||
if s, ok := p.(markers.Submitter); ok {
|
||||
submitter = s
|
||||
}
|
||||
}
|
||||
}
|
||||
if submitter == nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Provider does not support contribution")
|
||||
return
|
||||
}
|
||||
stats, err := submitter.FetchUserStats(r.Context())
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": true, "stats": toMarkerUserStatsResponse(stats)})
|
||||
}
|
||||
|
||||
func toProviderConfigResponse(c markers.ProviderConfig, isSubmitter bool, desc markers.ProviderDescriptor) providerConfigResponse {
|
||||
return providerConfigResponse{
|
||||
Provider: c.Provider,
|
||||
DisplayName: desc.DisplayName,
|
||||
SourceType: desc.SourceType,
|
||||
PluginID: desc.PluginID,
|
||||
PluginInstallationID: desc.PluginInstallationID,
|
||||
CapabilityID: desc.CapabilityID,
|
||||
IsSubmitter: isSubmitter,
|
||||
FetchEnabled: c.FetchEnabled,
|
||||
FetchPriority: c.FetchPriority,
|
||||
ContributeEnabled: c.ContributeEnabled,
|
||||
ContributeAutoLocal: c.ContributeAutoLocal,
|
||||
ContributeMinConfidence: c.ContributeMinConfidence,
|
||||
}
|
||||
}
|
||||
|
||||
func toMarkerUserStatsResponse(s markers.UserStats) markerUserStatsResponse {
|
||||
return markerUserStatsResponse{
|
||||
Total: s.Total,
|
||||
Accepted: s.Accepted,
|
||||
Pending: s.Pending,
|
||||
Rejected: s.Rejected,
|
||||
AcceptanceRate: s.AcceptanceRate,
|
||||
CurrentStreak: s.CurrentStreak,
|
||||
BestStreak: s.BestStreak,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
type fakeMarkerStatsSubmitter struct{}
|
||||
|
||||
func (fakeMarkerStatsSubmitter) ID() string { return "introdb" }
|
||||
func (fakeMarkerStatsSubmitter) FetchMarkers(context.Context, markers.Request) (markers.Result, error) {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) SubmitMarker(context.Context, markers.SubmissionRequest) (markers.SubmissionResult, error) {
|
||||
return markers.SubmissionResult{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) FetchUserStats(context.Context) (markers.UserStats, error) {
|
||||
return markers.UserStats{Total: 10, Accepted: 7, Pending: 2, Rejected: 1, AcceptanceRate: 0.7, CurrentStreak: 3, BestStreak: 5}, nil
|
||||
}
|
||||
|
||||
func TestValidateMarkerProviderUsesSnakeCaseStats(t *testing.T) {
|
||||
reg := markers.NewRegistry(nil)
|
||||
if err := reg.Register(fakeMarkerStatsSubmitter{}); err != nil {
|
||||
t.Fatalf("register provider: %v", err)
|
||||
}
|
||||
h := NewAdminMarkerProvidersHandler(reg, nil, nil, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/markers/providers/introdb/validate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("provider", "introdb")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleValidateProvider(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"acceptance_rate":0.7`) || strings.Contains(body, "AcceptanceRate") {
|
||||
t.Fatalf("unexpected stats response shape: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
type fakePluginMarkerSubmitter struct{ fakeMarkerStatsSubmitter }
|
||||
|
||||
func (fakePluginMarkerSubmitter) ID() string { return "plugin:6:introdb" }
|
||||
|
||||
// Browsers send encodeURIComponent-escaped provider IDs (plugin%3A6%3Aintrodb),
|
||||
// and chi matches the raw path, so the handler must decode the route param.
|
||||
func TestValidateMarkerProviderDecodesEncodedID(t *testing.T) {
|
||||
reg := markers.NewRegistry(nil)
|
||||
if err := reg.Register(fakePluginMarkerSubmitter{}); err != nil {
|
||||
t.Fatalf("register provider: %v", err)
|
||||
}
|
||||
h := NewAdminMarkerProvidersHandler(reg, nil, nil, nil)
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Post("/admin/markers/providers/{provider}/validate", h.HandleValidateProvider)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/markers/providers/plugin%3A6%3Aintrodb/validate", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), `"valid":true`) {
|
||||
t.Fatalf("unexpected response: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerProviderResponseIncludesPluginMetadata(t *testing.T) {
|
||||
resp := toProviderConfigResponse(
|
||||
markers.ProviderConfig{Provider: "plugin:4:markers", FetchEnabled: true, FetchPriority: 25},
|
||||
true,
|
||||
markers.ProviderDescriptor{
|
||||
DisplayName: "Plugin Markers",
|
||||
SourceType: markers.ProviderSourcePlugin,
|
||||
PluginID: "silo.plugin.markers",
|
||||
PluginInstallationID: 4,
|
||||
CapabilityID: "markers",
|
||||
},
|
||||
)
|
||||
if resp.DisplayName != "Plugin Markers" || resp.SourceType != markers.ProviderSourcePlugin {
|
||||
t.Fatalf("plugin metadata fields missing: %+v", resp)
|
||||
}
|
||||
if resp.PluginID != "silo.plugin.markers" ||
|
||||
resp.PluginInstallationID != 4 ||
|
||||
resp.CapabilityID != "markers" ||
|
||||
!resp.IsSubmitter {
|
||||
t.Fatalf("plugin identity fields missing: %+v", resp)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/notifications"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Minimal fake UserRepository for AdminHandler tests.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type fakeUserRepo struct {
|
||||
user *models.User
|
||||
getErr error
|
||||
}
|
||||
|
||||
func (f *fakeUserRepo) List(_ context.Context) ([]*models.User, error) { return nil, nil }
|
||||
|
||||
func (f *fakeUserRepo) Create(_ context.Context, _ models.CreateUserInput) (*models.User, error) {
|
||||
return f.user, nil
|
||||
}
|
||||
|
||||
func (f *fakeUserRepo) Update(_ context.Context, _ int, _ models.UpdateUserInput) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeUserRepo) Delete(_ context.Context, _ int) error { return nil }
|
||||
|
||||
func (f *fakeUserRepo) GetByID(_ context.Context, _ int) (*models.User, error) {
|
||||
return f.user, f.getErr
|
||||
}
|
||||
|
||||
// Compile-time check.
|
||||
var _ UserRepository = (*fakeUserRepo)(nil)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper: build AdminHandler with a realtime hub whose events the test can
|
||||
// observe. Password changes publish a domain event; the notifications
|
||||
// materializer (tested separately) turns it into the user's notification.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func newAdminHandlerWithHub(userRepo UserRepository) (*AdminHandler, *evt.Hub) {
|
||||
rh := notifications.NewHub("test", nil)
|
||||
h := NewAdminHandler(userRepo, nil, nil)
|
||||
h.RealtimeHub = rh
|
||||
return h, rh.EventsHub()
|
||||
}
|
||||
|
||||
// adminUpdateRequest builds a PUT /admin/users/{id} HTTP request.
|
||||
func adminUpdateRequest(body []byte, userID string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/admin/users/"+userID, bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", userID)
|
||||
return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: HandleUpdateUser password change notification
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TestHandleUpdateUser_WithPassword_PublishesEvent asserts that including a
|
||||
// Password in the update payload publishes one user.password_changed event on
|
||||
// ChannelSessions targeted at the updated user.
|
||||
func TestHandleUpdateUser_WithPassword_PublishesEvent(t *testing.T) {
|
||||
targetUser := &models.User{
|
||||
ID: 99,
|
||||
Username: "alice",
|
||||
Email: "alice@example.com",
|
||||
Role: "user",
|
||||
}
|
||||
repo := &fakeUserRepo{user: targetUser}
|
||||
h, eventsHub := newAdminHandlerWithHub(repo)
|
||||
ch, unsub := eventsHub.Subscribe()
|
||||
defer unsub()
|
||||
|
||||
body := []byte(`{"password":"newSecret123"}`)
|
||||
req := adminUpdateRequest(body, "99")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h.HandleUpdateUser(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case env := <-ch:
|
||||
if env.Channel != evt.ChannelSessions {
|
||||
t.Errorf("channel = %q, want %q", env.Channel, evt.ChannelSessions)
|
||||
}
|
||||
if env.Event != notifications.EventUserPasswordChanged {
|
||||
t.Errorf("event = %q, want %q", env.Event, notifications.EventUserPasswordChanged)
|
||||
}
|
||||
if env.UserID != 99 {
|
||||
t.Errorf("event user_id = %d, want 99", env.UserID)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("expected a user.password_changed event, got none")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleUpdateUser_WithoutPassword_NoEvent asserts that a non-password
|
||||
// update publishes no user.password_changed event.
|
||||
func TestHandleUpdateUser_WithoutPassword_NoEvent(t *testing.T) {
|
||||
targetUser := &models.User{
|
||||
ID: 99,
|
||||
Username: "alice",
|
||||
Email: "alice@example.com",
|
||||
Role: "user",
|
||||
}
|
||||
repo := &fakeUserRepo{user: targetUser}
|
||||
h, eventsHub := newAdminHandlerWithHub(repo)
|
||||
ch, unsub := eventsHub.Subscribe()
|
||||
defer unsub()
|
||||
|
||||
body := []byte(`{"username":"alice-renamed"}`)
|
||||
req := adminUpdateRequest(body, "99")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h.HandleUpdateUser(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case env := <-ch:
|
||||
t.Errorf("expected no event, got %q on %q", env.Event, env.Channel)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// no event, as expected
|
||||
}
|
||||
}
|
||||
@@ -326,6 +326,47 @@ func TestAdminUpdateSettingRedactsSensitiveSetting(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminUpdateSettingReportsRestartRequired(t *testing.T) {
|
||||
cases := []struct {
|
||||
key string
|
||||
value string
|
||||
restartRequired bool
|
||||
}{
|
||||
// Infrastructure settings are captured at startup.
|
||||
{key: "database.max_connections", value: "40", restartRequired: true},
|
||||
{key: "s3.public_bucket", value: "assets", restartRequired: true},
|
||||
// Branding is read live from the settings repo per request.
|
||||
{key: "branding.server_name", value: "Casa", restartRequired: false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.key, func(t *testing.T) {
|
||||
settings := &fakeServerSettingsStore{}
|
||||
handler := &AdminHandler{SettingsRepo: settings}
|
||||
|
||||
req := httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/admin/settings/"+tc.key,
|
||||
strings.NewReader(`{"value":"`+tc.value+`"}`),
|
||||
)
|
||||
req = withChiParam(req, "key", tc.key)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleUpdateSetting(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var resp adminSettingResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.RestartRequired != tc.restartRequired {
|
||||
t.Fatalf("restart_required = %v, want %v", resp.RestartRequired, tc.restartRequired)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func performSettingsCheckRequest(
|
||||
t *testing.T,
|
||||
handler *AdminHandler,
|
||||
|
||||
@@ -22,7 +22,9 @@ type AdminStats struct {
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalUsers int `json:"total_users"`
|
||||
TotalMovies int `json:"total_movies"`
|
||||
TotalMovieFiles int `json:"total_movie_files"`
|
||||
TotalShows int `json:"total_shows"`
|
||||
TotalShowFiles int `json:"total_show_files"`
|
||||
ActiveStreams int `json:"active_streams"`
|
||||
TotalStorageBytes int64 `json:"total_storage_bytes"`
|
||||
WatchProviderActivity WatchProviderActivity `json:"watch_provider_activity"`
|
||||
@@ -127,13 +129,15 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
}
|
||||
|
||||
var (
|
||||
totalUsers int64
|
||||
totalItems int64
|
||||
totalFiles int64
|
||||
totalMovies int64
|
||||
totalShows int64
|
||||
activeStreams int64
|
||||
totalStorage int64
|
||||
totalUsers int64
|
||||
totalItems int64
|
||||
totalFiles int64
|
||||
totalMovies int64
|
||||
totalMovieFiles int64
|
||||
totalShows int64
|
||||
totalShowFiles int64
|
||||
activeStreams int64
|
||||
totalStorage int64
|
||||
)
|
||||
|
||||
row := pool.QueryRow(ctx, `
|
||||
@@ -151,8 +155,34 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
file_stats AS (
|
||||
SELECT
|
||||
COUNT(*)::bigint AS total_files,
|
||||
COUNT(*) FILTER (WHERE file_kind = 'movie')::bigint AS total_movie_files,
|
||||
COUNT(*) FILTER (WHERE file_kind = 'series')::bigint AS total_show_files,
|
||||
COALESCE(SUM(file_size), 0)::bigint AS total_storage_bytes
|
||||
FROM media_files
|
||||
FROM (
|
||||
SELECT
|
||||
media_files.file_size,
|
||||
CASE
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_items.type, ''), ''))) = 'movie'
|
||||
THEN 'movie'
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_items.type, ''), ''))) = 'series'
|
||||
THEN 'series'
|
||||
WHEN episodes.content_id IS NOT NULL
|
||||
THEN 'series'
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_files.base_type, ''), ''))) IN ('movie', 'movies')
|
||||
THEN 'movie'
|
||||
WHEN lower(trim(COALESCE(NULLIF(media_files.base_type, ''), ''))) IN ('series', 'tv', 'show', 'shows', 'tvshows')
|
||||
THEN 'series'
|
||||
WHEN lower(trim(media_folders.type)) IN ('movie', 'movies')
|
||||
THEN 'movie'
|
||||
WHEN lower(trim(media_folders.type)) IN ('series', 'tv', 'show', 'shows', 'tvshows')
|
||||
THEN 'series'
|
||||
ELSE ''
|
||||
END AS file_kind
|
||||
FROM media_files
|
||||
JOIN media_folders ON media_folders.id = media_files.media_folder_id
|
||||
LEFT JOIN media_items ON media_items.content_id = media_files.content_id
|
||||
LEFT JOIN episodes ON episodes.content_id = media_files.episode_id
|
||||
) classified_files
|
||||
),
|
||||
session_stats AS (
|
||||
SELECT COUNT(*)::bigint AS active_streams
|
||||
@@ -163,7 +193,9 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
item_stats.total_items,
|
||||
file_stats.total_files,
|
||||
item_stats.total_movies,
|
||||
file_stats.total_movie_files,
|
||||
item_stats.total_shows,
|
||||
file_stats.total_show_files,
|
||||
session_stats.active_streams,
|
||||
file_stats.total_storage_bytes
|
||||
FROM user_stats
|
||||
@@ -176,7 +208,9 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
&totalItems,
|
||||
&totalFiles,
|
||||
&totalMovies,
|
||||
&totalMovieFiles,
|
||||
&totalShows,
|
||||
&totalShowFiles,
|
||||
&activeStreams,
|
||||
&totalStorage,
|
||||
); err != nil {
|
||||
@@ -194,7 +228,9 @@ func queryAdminStats(ctx context.Context, pool *pgxpool.Pool) (AdminStats, error
|
||||
TotalItems: int(totalItems),
|
||||
TotalFiles: int(totalFiles),
|
||||
TotalMovies: int(totalMovies),
|
||||
TotalMovieFiles: int(totalMovieFiles),
|
||||
TotalShows: int(totalShows),
|
||||
TotalShowFiles: int(totalShowFiles),
|
||||
ActiveStreams: int(activeStreams),
|
||||
TotalStorageBytes: totalStorage,
|
||||
WatchProviderActivity: activity,
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/opensubtitles"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/subdl"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/subsource"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
@@ -57,7 +56,11 @@ func (h *AdminSubtitleHandler) HandleListProviders(w http.ResponseWriter, r *htt
|
||||
|
||||
// HandleUpdateProvider handles PUT /api/v1/admin/subtitle-providers/{provider}
|
||||
func (h *AdminSubtitleHandler) HandleUpdateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
providerName := chi.URLParam(r, "provider")
|
||||
providerName, err := decodedURLParam(r, "provider")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid provider name")
|
||||
return
|
||||
}
|
||||
|
||||
var req updateSubtitleProviderRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
@@ -83,7 +86,11 @@ func (h *AdminSubtitleHandler) HandleUpdateProvider(w http.ResponseWriter, r *ht
|
||||
|
||||
// HandleTestProvider handles POST /api/v1/admin/subtitle-providers/{provider}/test
|
||||
func (h *AdminSubtitleHandler) HandleTestProvider(w http.ResponseWriter, r *http.Request) {
|
||||
providerName := chi.URLParam(r, "provider")
|
||||
providerName, err := decodedURLParam(r, "provider")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid provider name")
|
||||
return
|
||||
}
|
||||
|
||||
cfg, err := h.repo.GetProviderConfig(r.Context(), providerName)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,73 +1,132 @@
|
||||
package handlers
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestUpdateRequiresSessionRevocation(t *testing.T) {
|
||||
role := "admin"
|
||||
sameRole := "user"
|
||||
enabled := true
|
||||
disabled := false
|
||||
libraryIDs := []int{1, 2}
|
||||
sameLibraryIDs := []int{1}
|
||||
emptyLibraryIDs := []int{}
|
||||
var allLibraryIDs []int
|
||||
maxPlaybackQuality := "1080p"
|
||||
sameMaxPlaybackQuality := "original"
|
||||
password := "new-password"
|
||||
username := "renamed"
|
||||
maxStreams := 4
|
||||
permissions := []string{"metadata_curation"}
|
||||
samePermissions := []string{"download"}
|
||||
|
||||
current := &models.User{
|
||||
Role: "user",
|
||||
Permissions: []string{"download"},
|
||||
Enabled: false,
|
||||
LibraryIDs: []int{1},
|
||||
MaxPlaybackQuality: "original",
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
req updateUserRequest
|
||||
in models.UpdateUserInput
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "permissions set",
|
||||
req: updateUserRequest{Permissions: updateStringSliceField{Set: true, Value: []string{"metadata_curation"}}},
|
||||
in: models.UpdateUserInput{Permissions: &permissions},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "permissions unset",
|
||||
req: updateUserRequest{Permissions: updateStringSliceField{Set: false, Value: []string{"metadata_curation"}}},
|
||||
name: "permissions unchanged",
|
||||
in: models.UpdateUserInput{Permissions: &samePermissions},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "role",
|
||||
req: updateUserRequest{Role: &role},
|
||||
in: models.UpdateUserInput{Role: &role},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "role unchanged",
|
||||
in: models.UpdateUserInput{Role: &sameRole},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "enabled",
|
||||
req: updateUserRequest{Enabled: &enabled},
|
||||
in: models.UpdateUserInput{Enabled: &enabled},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "library ids",
|
||||
req: updateUserRequest{LibraryIDs: updateLibraryIDsField{Set: true, Value: libraryIDs}},
|
||||
want: true,
|
||||
name: "enabled unchanged",
|
||||
in: models.UpdateUserInput{Enabled: &disabled},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "library ids does not revoke session",
|
||||
in: models.UpdateUserInput{LibraryIDs: &libraryIDs},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "library ids unchanged",
|
||||
in: models.UpdateUserInput{LibraryIDs: &sameLibraryIDs},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "library ids nil does not revoke session",
|
||||
in: models.UpdateUserInput{LibraryIDs: &allLibraryIDs},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "max playback quality",
|
||||
req: updateUserRequest{MaxPlaybackQuality: &maxPlaybackQuality},
|
||||
in: models.UpdateUserInput{MaxPlaybackQuality: &maxPlaybackQuality},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "max playback quality unchanged",
|
||||
in: models.UpdateUserInput{MaxPlaybackQuality: &sameMaxPlaybackQuality},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "password",
|
||||
req: updateUserRequest{Password: &password},
|
||||
in: models.UpdateUserInput{Password: &password},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "non access fields",
|
||||
req: updateUserRequest{Username: &username, MaxStreams: &maxStreams},
|
||||
in: models.UpdateUserInput{Username: &username, MaxStreams: &maxStreams},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty update",
|
||||
req: updateUserRequest{},
|
||||
in: models.UpdateUserInput{},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := updateRequiresSessionRevocation(tt.req); got != tt.want {
|
||||
if got := updateRequiresSessionRevocation(current, tt.in); got != tt.want {
|
||||
t.Fatalf("updateRequiresSessionRevocation() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
unrestrictedCurrent := *current
|
||||
unrestrictedCurrent.LibraryIDs = nil
|
||||
t.Run("library ids empty does not revoke session", func(t *testing.T) {
|
||||
if got := updateRequiresSessionRevocation(&unrestrictedCurrent, models.UpdateUserInput{LibraryIDs: &emptyLibraryIDs}); got {
|
||||
t.Fatalf("updateRequiresSessionRevocation() = %v, want false", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("library ids nil unchanged", func(t *testing.T) {
|
||||
if got := updateRequiresSessionRevocation(&unrestrictedCurrent, models.UpdateUserInput{LibraryIDs: &allLibraryIDs}); got {
|
||||
t.Fatalf("updateRequiresSessionRevocation() = %v, want false", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
)
|
||||
|
||||
type audiobookGroupResponse struct {
|
||||
Name string `json:"name"`
|
||||
ItemCount int `json:"item_count"`
|
||||
TotalDurationSeconds int64 `json:"total_duration_seconds"`
|
||||
InProgressCount int `json:"in_progress_count"`
|
||||
FinishedCount int `json:"finished_count"`
|
||||
PosterURLs []string `json:"poster_urls"`
|
||||
}
|
||||
|
||||
type audiobookGroupsResponse struct {
|
||||
Total int `json:"total"`
|
||||
Groups []audiobookGroupResponse `json:"groups"`
|
||||
}
|
||||
|
||||
// HandleGetAudiobookGroups — GET /api/v1/catalog/audiobook-groups
|
||||
//
|
||||
// Grouped browse for audiobook libraries: authors, narrators, or series with
|
||||
// aggregate stats (book count, total duration, per-profile progress counts)
|
||||
// and up to four poster URLs for cover stacks. Query parameters:
|
||||
// library_id=<id> (required), group_by=author|narrator|series (required),
|
||||
// sort=name|count|duration (default name), limit, offset.
|
||||
//
|
||||
// Group names round-trip into the corresponding catalog filter fields
|
||||
// (author / narrator / series), which match case-insensitively.
|
||||
func (h *CatalogHandler) HandleGetAudiobookGroups(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.itemsH == nil || h.itemsH.browseRepo == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Catalog is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
libraryID, err := strconv.Atoi(strings.TrimSpace(r.URL.Query().Get("library_id")))
|
||||
if err != nil || libraryID <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "library_id must be a positive integer")
|
||||
return
|
||||
}
|
||||
|
||||
groupBy, ok := catalog.ParseAudiobookGroupBy(r.URL.Query().Get("group_by"))
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "group_by must be one of author, narrator, series")
|
||||
return
|
||||
}
|
||||
|
||||
sort := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("sort")))
|
||||
switch sort {
|
||||
case "", "name", "count", "duration":
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "sort must be one of name, count, duration")
|
||||
return
|
||||
}
|
||||
|
||||
limit := 200
|
||||
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
|
||||
n, parseErr := strconv.Atoi(raw)
|
||||
if parseErr != nil || n <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "limit must be a positive integer")
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
offset := max(catalog.ParseIntParam(r.URL.Query().Get("offset")), 0)
|
||||
|
||||
groups, total, err := catalog.ListAudiobookGroups(
|
||||
r.Context(),
|
||||
h.itemsH.browseRepo.Pool(),
|
||||
catalog.AudiobookGroupsQuery{
|
||||
LibraryID: libraryID,
|
||||
GroupBy: groupBy,
|
||||
Sort: sort,
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
},
|
||||
h.itemsH.accessFilter(r),
|
||||
)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list audiobook groups")
|
||||
return
|
||||
}
|
||||
|
||||
resp := audiobookGroupsResponse{Total: total, Groups: make([]audiobookGroupResponse, 0, len(groups))}
|
||||
for _, g := range groups {
|
||||
posterURLs := make([]string, 0, len(g.PosterPaths))
|
||||
for _, path := range g.PosterPaths {
|
||||
if url := h.itemsH.presignURL(r, cardThumbnailPath(path), "card"); url != "" {
|
||||
posterURLs = append(posterURLs, url)
|
||||
}
|
||||
}
|
||||
resp.Groups = append(resp.Groups, audiobookGroupResponse{
|
||||
Name: g.Name,
|
||||
ItemCount: g.ItemCount,
|
||||
TotalDurationSeconds: g.TotalDurationSeconds,
|
||||
InProgressCount: g.InProgressCount,
|
||||
FinishedCount: g.FinishedCount,
|
||||
PosterURLs: posterURLs,
|
||||
})
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
@@ -1,532 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// HandleGetAudiobook serves GET /api/v1/audiobooks/{id}. Returns the
|
||||
// audiobook media_item, its media_files (with embedded chapters),
|
||||
// author/narrator from item_people (kinds 7/8), and the caller's
|
||||
// per-profile listening progress.
|
||||
func (h *AudiobookHandler) HandleGetAudiobook(w http.ResponseWriter, r *http.Request) {
|
||||
contentID := chi.URLParam(r, "id")
|
||||
if contentID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "id is required")
|
||||
return
|
||||
}
|
||||
|
||||
item, err := h.Items.GetByID(r.Context(), contentID)
|
||||
if err != nil {
|
||||
if errors.Is(err, catalog.ErrItemNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load audiobook failed")
|
||||
return
|
||||
}
|
||||
if item == nil || item.Type != "audiobook" {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByContentID(r.Context(), contentID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load files failed")
|
||||
return
|
||||
}
|
||||
|
||||
author, narrator, err := h.fetchAuthorNarrator(r, contentID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load people failed")
|
||||
return
|
||||
}
|
||||
|
||||
progress := h.fetchProgress(r, contentID)
|
||||
|
||||
alsoByAuthor := h.fetchAlsoByAuthor(r.Context(), contentID)
|
||||
inSeries := h.fetchInSeries(r.Context(), contentID)
|
||||
similar := h.fetchSimilar(r.Context(), contentID)
|
||||
otherNarrations := h.fetchOtherNarrations(r.Context(), contentID)
|
||||
|
||||
resp := audiobookDetailResponse{
|
||||
Audiobook: audiobookDetailItem{
|
||||
ContentID: item.ContentID,
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterURL: h.presignPoster(r.Context(), item.PosterPath),
|
||||
Publisher: pickFirstString(item.Studios),
|
||||
Genres: append([]string(nil), item.Genres...),
|
||||
},
|
||||
Author: author,
|
||||
Narrator: narrator,
|
||||
Files: audiobookDetailFiles(files),
|
||||
Progress: progress,
|
||||
AlsoByAuthor: alsoByAuthor,
|
||||
InSeries: inSeries,
|
||||
SimilarAudiobooks: similar,
|
||||
OtherNarrations: otherNarrations,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// fetchAuthorNarrator retrieves the first author (kind=7) and narrator (kind=8)
|
||||
// name from item_people for the given content ID.
|
||||
func (h *AudiobookHandler) fetchAuthorNarrator(r *http.Request, contentID string) (author, narrator string, err error) {
|
||||
people, err := h.Items.GetPeople(r.Context(), contentID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
for _, p := range people {
|
||||
switch p.Kind {
|
||||
case models.PersonKindAuthor:
|
||||
if author == "" {
|
||||
author = p.Name
|
||||
}
|
||||
case models.PersonKindNarrator:
|
||||
if narrator == "" {
|
||||
narrator = p.Name
|
||||
}
|
||||
}
|
||||
if author != "" && narrator != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
return author, narrator, nil
|
||||
}
|
||||
|
||||
// fetchAlsoByAuthor returns other audiobooks sharing any author (item_people
|
||||
// kind=PersonKindAuthor) with the given book. Capped at 12 to keep the rail
|
||||
// snappy on the client. Returns an empty slice when there is no author or
|
||||
// no other matching titles.
|
||||
func (h *AudiobookHandler) fetchAlsoByAuthor(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
const q = `
|
||||
SELECT m2.content_id, m2.title, COALESCE(m2.year, 0), COALESCE(m2.poster_path, '')
|
||||
FROM item_people ip1
|
||||
JOIN item_people ip2
|
||||
ON ip2.person_id = ip1.person_id
|
||||
AND ip2.kind = ip1.kind
|
||||
AND ip2.content_id <> ip1.content_id
|
||||
JOIN media_items m2
|
||||
ON m2.content_id = ip2.content_id
|
||||
AND m2.type = 'audiobook'
|
||||
WHERE ip1.content_id = $1
|
||||
AND ip1.kind = $2
|
||||
ORDER BY m2.year DESC NULLS LAST, LOWER(m2.sort_title)
|
||||
LIMIT 12
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID, models.PersonKindAuthor)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]audiobookRelatedItem, 0, 12)
|
||||
seen := make(map[string]struct{})
|
||||
for rows.Next() {
|
||||
var it audiobookRelatedItem
|
||||
var poster string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster); err != nil {
|
||||
return nil
|
||||
}
|
||||
if _, dup := seen[it.ContentID]; dup {
|
||||
continue
|
||||
}
|
||||
seen[it.ContentID] = struct{}{}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
out = append(out, it)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchSimilar returns the "You might also like" rail. Prefers embedding-based
|
||||
// nearest-neighbor search when the source book has an embedding; falls back to
|
||||
// shared-genre matching otherwise (book is brand new, embedding job hasn't run,
|
||||
// or the embedding lookup errored). The genre fallback excludes same-author
|
||||
// books so the rail complements rather than duplicates also_by_author.
|
||||
func (h *AudiobookHandler) fetchSimilar(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Recs != nil {
|
||||
if items := h.fetchSimilarByEmbedding(ctx, contentID); items != nil {
|
||||
return items
|
||||
}
|
||||
}
|
||||
return h.fetchSimilarByGenres(ctx, contentID)
|
||||
}
|
||||
|
||||
// fetchSimilarByEmbedding returns nearest-neighbor audiobooks by cosine
|
||||
// similarity on the gemini embedding vector. Returns nil (not an empty slice)
|
||||
// when the source book has no embedding yet, so the caller can fall back to
|
||||
// the genre-based recommender.
|
||||
func (h *AudiobookHandler) fetchSimilarByEmbedding(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Recs == nil || h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
emb, err := h.Recs.GetEmbedding(ctx, contentID)
|
||||
if err != nil || emb == nil {
|
||||
return nil
|
||||
}
|
||||
// Pull extra candidates so we can drop the source book's own author from
|
||||
// the rail (they belong in also_by_author).
|
||||
scored, err := h.Recs.FindSimilar(ctx, emb, []string{contentID}, "audiobook", 36)
|
||||
if err != nil || len(scored) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]string, 0, len(scored))
|
||||
for _, s := range scored {
|
||||
ids = append(ids, s.MediaItemID)
|
||||
}
|
||||
const q = `
|
||||
WITH this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $1 AND kind = $2
|
||||
)
|
||||
SELECT m.content_id, m.title, COALESCE(m.year, 0), COALESCE(m.poster_path, '')
|
||||
FROM media_items m
|
||||
WHERE m.content_id = ANY($3)
|
||||
AND m.type = 'audiobook'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $2
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID, models.PersonKindAuthor, ids)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
byID := make(map[string]audiobookRelatedItem, len(scored))
|
||||
for rows.Next() {
|
||||
var it audiobookRelatedItem
|
||||
var poster string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster); err != nil {
|
||||
return nil
|
||||
}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
byID[it.ContentID] = it
|
||||
}
|
||||
out := make([]audiobookRelatedItem, 0, 12)
|
||||
for _, s := range scored {
|
||||
if it, ok := byID[s.MediaItemID]; ok {
|
||||
out = append(out, it)
|
||||
if len(out) >= 12 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchOtherNarrations returns sibling audiobook items that share the same
|
||||
// "core" title (after stripping the common "read by X" / "(Read by X)" /
|
||||
// "(UK Version: Read by X)" suffixes) and the same author. Powers the
|
||||
// narrator picker dropdown on the detail page when a single book has
|
||||
// multiple narration editions stored as separate items in the catalog.
|
||||
func (h *AudiobookHandler) fetchOtherNarrations(ctx context.Context, contentID string) []audiobookNarration {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
// The regex strips a trailing block of the form "[ separator] [( ]read by ...[) ]".
|
||||
// Tested against patterns in the corpus: "Foo Read By X",
|
||||
// "Foo - read by X", "Foo (Read by X)", "Foo (UK Version: Read by X)".
|
||||
const stripRE = `\s*\(?\s*[-:,]?\s*(UK Version:?|US Version:?)?\s*[Rr]ead [Bb]y [^()]+\)?\s*$`
|
||||
const q = `
|
||||
WITH this_book AS (
|
||||
SELECT trim(regexp_replace(title, $1, '', 'g')) AS norm_title
|
||||
FROM media_items WHERE content_id = $2
|
||||
),
|
||||
this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $2 AND kind = $3
|
||||
)
|
||||
SELECT
|
||||
m.content_id,
|
||||
m.title,
|
||||
COALESCE(MAX(p.name), '') AS narrator,
|
||||
COALESCE(m.year, 0)
|
||||
FROM media_items m
|
||||
LEFT JOIN item_people ipn ON ipn.content_id = m.content_id AND ipn.kind = $4
|
||||
LEFT JOIN people p ON p.id = ipn.person_id
|
||||
WHERE m.type = 'audiobook'
|
||||
AND m.content_id <> $2
|
||||
AND trim(regexp_replace(m.title, $1, '', 'g')) = (SELECT norm_title FROM this_book)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $3
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)
|
||||
GROUP BY m.content_id, m.title, m.year
|
||||
ORDER BY m.year DESC NULLS LAST, m.title
|
||||
LIMIT 8
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q,
|
||||
stripRE, contentID, models.PersonKindAuthor, models.PersonKindNarrator)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]audiobookNarration, 0, 4)
|
||||
for rows.Next() {
|
||||
var n audiobookNarration
|
||||
if err := rows.Scan(&n.ContentID, &n.Title, &n.Narrator, &n.Year); err != nil {
|
||||
return nil
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchSimilarByGenres returns up to 12 audiobooks sharing genres with the
|
||||
// given book, excluding books by the same author so the rail complements
|
||||
// rather than duplicates also_by_author. Used as a fallback when no
|
||||
// embedding exists for the source yet.
|
||||
func (h *AudiobookHandler) fetchSimilarByGenres(ctx context.Context, contentID string) []audiobookRelatedItem {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
const q = `
|
||||
WITH this_genres AS (
|
||||
SELECT unnest(genres) AS g FROM media_items WHERE content_id = $1
|
||||
),
|
||||
this_author AS (
|
||||
SELECT person_id FROM item_people WHERE content_id = $1 AND kind = $2
|
||||
)
|
||||
SELECT m.content_id, m.title, COALESCE(m.year, 0), COALESCE(m.poster_path, '')
|
||||
FROM media_items m
|
||||
WHERE m.type = 'audiobook'
|
||||
AND m.content_id <> $1
|
||||
AND m.genres && (SELECT array_agg(g) FROM this_genres)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM item_people ip
|
||||
WHERE ip.content_id = m.content_id
|
||||
AND ip.kind = $2
|
||||
AND ip.person_id IN (SELECT person_id FROM this_author)
|
||||
)
|
||||
ORDER BY
|
||||
cardinality(ARRAY(SELECT unnest(m.genres) INTERSECT SELECT g FROM this_genres)) DESC,
|
||||
COALESCE(m.year, 0) DESC,
|
||||
LOWER(m.sort_title)
|
||||
LIMIT 12
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID, models.PersonKindAuthor)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]audiobookRelatedItem, 0, 12)
|
||||
for rows.Next() {
|
||||
var it audiobookRelatedItem
|
||||
var poster string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster); err != nil {
|
||||
return nil
|
||||
}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
out = append(out, it)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fetchInSeries returns the series the audiobook belongs to (if any) along
|
||||
// with the ordered list of sibling entries. Driven by the audiobook_series
|
||||
// table populated by either the scanner (tag-derived) or migration 145's
|
||||
// best-effort title-pattern backfill.
|
||||
func (h *AudiobookHandler) fetchInSeries(ctx context.Context, contentID string) *audiobookSeriesGroup {
|
||||
if h.Files == nil {
|
||||
return nil
|
||||
}
|
||||
const q = `
|
||||
SELECT
|
||||
m.content_id,
|
||||
m.title,
|
||||
COALESCE(m.year, 0),
|
||||
COALESCE(m.poster_path, ''),
|
||||
s.series_name,
|
||||
s.series_index
|
||||
FROM audiobook_series root
|
||||
JOIN audiobook_series s ON LOWER(s.series_name) = LOWER(root.series_name)
|
||||
JOIN media_items m ON m.content_id = s.content_id AND m.type = 'audiobook'
|
||||
WHERE root.content_id = $1
|
||||
ORDER BY s.series_index NULLS LAST, LOWER(m.sort_title)
|
||||
LIMIT 30
|
||||
`
|
||||
rows, err := h.Files.Pool().Query(ctx, q, contentID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var seriesName string
|
||||
entries := make([]audiobookRelatedItem, 0, 16)
|
||||
for rows.Next() {
|
||||
var (
|
||||
it audiobookRelatedItem
|
||||
poster string
|
||||
thisName string
|
||||
idxNum *float64
|
||||
)
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &poster, &thisName, &idxNum); err != nil {
|
||||
return nil
|
||||
}
|
||||
if seriesName == "" {
|
||||
seriesName = thisName
|
||||
}
|
||||
if idxNum != nil {
|
||||
n := int(*idxNum)
|
||||
if float64(n) == *idxNum {
|
||||
it.SeriesIndex = &n
|
||||
}
|
||||
}
|
||||
it.PosterURL = h.presignPoster(ctx, poster)
|
||||
entries = append(entries, it)
|
||||
}
|
||||
// A series with only this book is not useful as a rail.
|
||||
if len(entries) < 2 {
|
||||
return nil
|
||||
}
|
||||
return &audiobookSeriesGroup{Name: seriesName, Entries: entries}
|
||||
}
|
||||
|
||||
// fetchProgress returns the caller's listening progress for this audiobook, or
|
||||
// nil when no progress exists or the caller is not authenticated with a profile.
|
||||
func (h *AudiobookHandler) fetchProgress(r *http.Request, contentID string) *audiobookDetailProgress {
|
||||
if h.StoreProvider == nil {
|
||||
return nil
|
||||
}
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
if userID == 0 {
|
||||
return nil
|
||||
}
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
if profileID == "" {
|
||||
profileID = r.Header.Get("X-Profile-Id")
|
||||
}
|
||||
if profileID == "" {
|
||||
return nil
|
||||
}
|
||||
store, err := h.StoreProvider.ForUser(r.Context(), userID)
|
||||
if err != nil || store == nil {
|
||||
return nil
|
||||
}
|
||||
wp, err := store.GetProgress(r.Context(), profileID, contentID)
|
||||
if err != nil || wp == nil {
|
||||
return nil
|
||||
}
|
||||
return &audiobookDetailProgress{
|
||||
PositionSeconds: wp.PositionSeconds,
|
||||
Completed: wp.Completed,
|
||||
UpdatedAt: wp.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
type audiobookDetailResponse struct {
|
||||
Audiobook audiobookDetailItem `json:"audiobook"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Narrator string `json:"narrator,omitempty"`
|
||||
Files []audiobookDetailFile `json:"files"`
|
||||
Progress *audiobookDetailProgress `json:"progress,omitempty"`
|
||||
AlsoByAuthor []audiobookRelatedItem `json:"also_by_author,omitempty"`
|
||||
InSeries *audiobookSeriesGroup `json:"in_series,omitempty"`
|
||||
SimilarAudiobooks []audiobookRelatedItem `json:"similar_audiobooks,omitempty"`
|
||||
OtherNarrations []audiobookNarration `json:"other_narrations,omitempty"`
|
||||
}
|
||||
|
||||
// audiobookNarration is a sibling narration (same book, different
|
||||
// narrator/edition). Returned alongside the primary item so the UI can
|
||||
// surface a narrator picker.
|
||||
type audiobookNarration struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Narrator string `json:"narrator,omitempty"`
|
||||
Year int `json:"year,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookRelatedItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
SeriesIndex *int `json:"series_index,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookSeriesGroup struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Entries []audiobookRelatedItem `json:"entries"`
|
||||
}
|
||||
|
||||
type audiobookDetailItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
Publisher string `json:"publisher,omitempty"`
|
||||
Genres []string `json:"genres,omitempty"`
|
||||
}
|
||||
|
||||
func (h *AudiobookHandler) presignPoster(ctx context.Context, path string) string {
|
||||
if path == "" || h.Detail == nil {
|
||||
return path
|
||||
}
|
||||
return h.Detail.PresignURL(ctx, path, "featured")
|
||||
}
|
||||
|
||||
func pickFirstString(values []string) string {
|
||||
for _, v := range values {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type audiobookDetailFile struct {
|
||||
ID int `json:"id"`
|
||||
Path string `json:"path"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
AudioChannels int `json:"audio_channels,omitempty"`
|
||||
Chapters []models.MediaChapter `json:"chapters,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookDetailProgress struct {
|
||||
PositionSeconds float64 `json:"position_seconds"`
|
||||
Completed bool `json:"completed"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func audiobookDetailFiles(files []*models.MediaFile) []audiobookDetailFile {
|
||||
out := make([]audiobookDetailFile, 0, len(files))
|
||||
for _, f := range files {
|
||||
out = append(out, audiobookDetailFile{
|
||||
ID: f.ID,
|
||||
Path: f.FilePath,
|
||||
DurationSeconds: f.Duration,
|
||||
Container: f.Container,
|
||||
CodecAudio: f.CodecAudio,
|
||||
Bitrate: f.Bitrate,
|
||||
AudioChannels: f.AudioChannels,
|
||||
Chapters: f.Chapters,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
// AudiobookHandler serves silo-native audiobook endpoints under
|
||||
// /api/v1/audiobooks/*. Sub-plan 3 implements list/detail/progress;
|
||||
// sub-plan 4's ABS-compat layer is separate.
|
||||
type AudiobookHandler struct {
|
||||
Items *catalog.ItemRepository
|
||||
Files *scanner.FileRepository
|
||||
StoreProvider userstore.UserStoreProvider
|
||||
Detail *catalog.DetailService
|
||||
// Recs is optional. When set, the detail handler uses embedding-based
|
||||
// nearest-neighbor search for the "You might also like" rail and falls
|
||||
// back to shared-genre matching only when no embedding exists for the
|
||||
// source book yet.
|
||||
Recs *recommendations.Repo
|
||||
}
|
||||
|
||||
// HandleListAudiobooks serves GET /api/v1/audiobooks?limit=&offset=.
|
||||
// Paginated catalog of media_items with type='audiobook'.
|
||||
func (h *AudiobookHandler) HandleListAudiobooks(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
genre := r.URL.Query().Get("genre")
|
||||
|
||||
pool := h.Files.Pool()
|
||||
ctx := r.Context()
|
||||
var total int
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM media_items
|
||||
WHERE type='audiobook' AND ($1 = '' OR $1 = ANY(genres))
|
||||
`, genre).Scan(&total); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks count failed")
|
||||
return
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT
|
||||
mi.content_id,
|
||||
mi.title,
|
||||
COALESCE(mi.year, 0),
|
||||
COALESCE(mi.poster_path, ''),
|
||||
COALESCE(SUM(mf.duration), 0)::int AS duration_seconds,
|
||||
COALESCE(MIN(mf.container), '') AS container,
|
||||
COALESCE(MIN(mf.codec_audio), '') AS codec_audio,
|
||||
COUNT(mf.id) AS file_count
|
||||
FROM media_items mi
|
||||
LEFT JOIN media_files mf ON mf.content_id = mi.content_id
|
||||
WHERE mi.type='audiobook'
|
||||
AND ($3 = '' OR $3 = ANY(mi.genres))
|
||||
GROUP BY mi.content_id, mi.title, mi.year, mi.poster_path, mi.sort_title
|
||||
ORDER BY LOWER(mi.sort_title), LOWER(mi.title)
|
||||
LIMIT $1 OFFSET $2
|
||||
`, limit, offset, genre)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks failed")
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]audiobookListItem, 0, limit)
|
||||
for rows.Next() {
|
||||
var it audiobookListItem
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &it.PosterURL, &it.DurationSeconds, &it.Container, &it.CodecAudio, &it.FileCount); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "scan audiobook row failed")
|
||||
return
|
||||
}
|
||||
if h.Detail != nil && it.PosterURL != "" {
|
||||
it.PosterURL = h.Detail.PresignURL(ctx, it.PosterURL, "card")
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "iterate audiobook rows failed")
|
||||
return
|
||||
}
|
||||
resp := struct {
|
||||
Items []audiobookListItem `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}{Items: items, Total: total, Limit: limit, Offset: offset}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
type audiobookListItem struct {
|
||||
ContentID string `json:"content_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
FileCount int `json:"file_count"`
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
type audiobookProgressRequest struct {
|
||||
PositionSeconds float64 `json:"position_seconds"`
|
||||
MediaFileID int `json:"media_file_id,omitempty"`
|
||||
}
|
||||
|
||||
// HandleReportAudiobookProgress serves POST /api/v1/audiobooks/{id}/progress.
|
||||
// UPSERTs into user_watch_progress for the caller's (user_id, profile_id, content_id).
|
||||
// Duration is unknown at the audiobook level (it lives on the media_file), so we
|
||||
// pass 0 and let the store compute completion via thresholds against position only.
|
||||
func (h *AudiobookHandler) HandleReportAudiobookProgress(w http.ResponseWriter, r *http.Request) {
|
||||
contentID := chi.URLParam(r, "id")
|
||||
if contentID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "id is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req audiobookProgressRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "invalid request body")
|
||||
return
|
||||
}
|
||||
if req.PositionSeconds < 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "position_seconds must be >= 0")
|
||||
return
|
||||
}
|
||||
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
profileID := apimw.GetProfileID(r.Context())
|
||||
if profileID == "" {
|
||||
profileID = r.Header.Get("X-Profile-Id")
|
||||
}
|
||||
if userID == 0 || profileID == "" {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "authentication required")
|
||||
return
|
||||
}
|
||||
|
||||
if h.StoreProvider == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "store unavailable")
|
||||
return
|
||||
}
|
||||
store, err := h.StoreProvider.ForUser(r.Context(), userID)
|
||||
if err != nil || store == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "store unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
// Duration is not carried in the progress body; pass 0 so the store
|
||||
// uses position-only heuristics. ProgressThresholds zero value means
|
||||
// "use defaults" (90% watched, 5% min-resume).
|
||||
if err := store.SetProgress(r.Context(), profileID, contentID, req.PositionSeconds, 0, userstore.ProgressThresholds{}); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "save progress failed")
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -17,9 +17,10 @@ import (
|
||||
|
||||
// AuthHandler handles authentication-related HTTP endpoints.
|
||||
type AuthHandler struct {
|
||||
service *auth.Service
|
||||
jwt *auth.JWTService
|
||||
device *auth.DeviceLoginService
|
||||
service *auth.Service
|
||||
jwt *auth.JWTService
|
||||
device *auth.DeviceLoginService
|
||||
oauthRoutesAvailable bool
|
||||
}
|
||||
|
||||
// NewAuthHandler creates a new AuthHandler backed by the given auth, JWT,
|
||||
@@ -32,6 +33,13 @@ func NewAuthHandler(service *auth.Service, jwt *auth.JWTService, device *auth.De
|
||||
}
|
||||
}
|
||||
|
||||
// SetOAuthRoutesAvailable controls whether OAuth login providers are
|
||||
// advertised by /auth/providers. The router only mounts OAuth routes when the
|
||||
// server has enough configuration to complete the flow.
|
||||
func (h *AuthHandler) SetOAuthRoutesAvailable(available bool) {
|
||||
h.oauthRoutesAvailable = available
|
||||
}
|
||||
|
||||
// --- Request/Response types ---
|
||||
|
||||
// loginRequest represents the JSON body of a POST /auth/login request.
|
||||
@@ -181,6 +189,9 @@ func (h *AuthHandler) HandleProviders(w http.ResponseWriter, r *http.Request) {
|
||||
providers := h.service.ListProviders()
|
||||
response := make([]authProviderResponse, 0, len(providers))
|
||||
for _, provider := range providers {
|
||||
if provider.Mode == "oauth" && !h.oauthRoutesAvailable {
|
||||
continue
|
||||
}
|
||||
response = append(response, authProviderResponse{
|
||||
ID: provider.ID,
|
||||
DisplayName: provider.DisplayName,
|
||||
@@ -214,6 +225,9 @@ func (h *AuthHandler) HandleSetup(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
req.Username = auth.NormalizeUsername(req.Username)
|
||||
req.Email = auth.NormalizeEmail(req.Email)
|
||||
|
||||
if req.Username == "" || req.Email == "" || req.Password == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Username, email, and password are required")
|
||||
return
|
||||
@@ -447,6 +461,9 @@ func (h *AuthHandler) HandleSignup(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
req.Username = auth.NormalizeUsername(req.Username)
|
||||
req.Email = auth.NormalizeEmail(req.Email)
|
||||
|
||||
if req.Username == "" || req.Email == "" || req.Password == "" || req.InviteCode == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Username, email, password, and invite code are required")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type stubLoginProvider struct{}
|
||||
|
||||
func (stubLoginProvider) Authenticate(context.Context, auth.Credentials) (*models.User, error) {
|
||||
return nil, auth.ErrInvalidCredentials
|
||||
}
|
||||
|
||||
func (stubLoginProvider) ValidateSession(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func newAuthProviderHandlerForTest(oauthRoutesAvailable bool) *AuthHandler {
|
||||
service := auth.NewService(nil, nil, nil, nil, nil, nil, nil)
|
||||
provider := stubLoginProvider{}
|
||||
|
||||
service.RegisterProvider(auth.LoginProviderInfo{
|
||||
ID: "local",
|
||||
DisplayName: "Local",
|
||||
Mode: "credentials",
|
||||
Default: true,
|
||||
}, provider)
|
||||
service.RegisterProvider(auth.LoginProviderInfo{
|
||||
ID: "plugin:41:oidc",
|
||||
DisplayName: "OIDC",
|
||||
Mode: "oauth",
|
||||
InstallationID: 41,
|
||||
}, provider)
|
||||
|
||||
handler := NewAuthHandler(service, nil, nil)
|
||||
handler.SetOAuthRoutesAvailable(oauthRoutesAvailable)
|
||||
return handler
|
||||
}
|
||||
|
||||
func readProviderResponse(t *testing.T, handler *AuthHandler) []authProviderResponse {
|
||||
t.Helper()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/auth/providers", nil)
|
||||
handler.HandleProviders(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
var providers []authProviderResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&providers); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
return providers
|
||||
}
|
||||
|
||||
func TestAuthProvidersHideOAuthWhenOAuthRoutesUnavailable(t *testing.T) {
|
||||
providers := readProviderResponse(t, newAuthProviderHandlerForTest(false))
|
||||
|
||||
if len(providers) != 1 {
|
||||
t.Fatalf("provider count = %d, want 1: %#v", len(providers), providers)
|
||||
}
|
||||
if providers[0].ID != "local" {
|
||||
t.Fatalf("provider ID = %q, want local", providers[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthProvidersIncludeOAuthWhenOAuthRoutesAvailable(t *testing.T) {
|
||||
providers := readProviderResponse(t, newAuthProviderHandlerForTest(true))
|
||||
|
||||
if len(providers) != 2 {
|
||||
t.Fatalf("provider count = %d, want 2: %#v", len(providers), providers)
|
||||
}
|
||||
|
||||
var foundOAuth bool
|
||||
for _, provider := range providers {
|
||||
if provider.ID == "plugin:41:oidc" {
|
||||
foundOAuth = true
|
||||
if provider.Mode != "oauth" {
|
||||
t.Fatalf("OAuth provider mode = %q, want oauth", provider.Mode)
|
||||
}
|
||||
if provider.InstallationID != 41 {
|
||||
t.Fatalf("OAuth provider installation ID = %d, want 41", provider.InstallationID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundOAuth {
|
||||
t.Fatalf("OAuth provider missing from response: %#v", providers)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,26 +2,69 @@ package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
)
|
||||
|
||||
type calendarRepository interface {
|
||||
ListEvents(ctx context.Context, f catalog.CalendarFilter) ([]catalog.CalendarEvent, error)
|
||||
}
|
||||
|
||||
// calendarPersonalRepo resolves per-profile id-sets and watched status.
|
||||
type calendarPersonalRepo interface {
|
||||
ListFollowedItemIDs(ctx context.Context, userID int, profileID string) ([]string, error)
|
||||
ListFavoriteItemIDs(ctx context.Context, userID int, profileID string) ([]string, error)
|
||||
ListWatchlistItemIDs(ctx context.Context, userID int, profileID string) ([]string, error)
|
||||
ListWatchedItemIDs(ctx context.Context, userID int, profileID string, contentIDs []string) (map[string]bool, error)
|
||||
}
|
||||
|
||||
// calendarPopularSource reads the cached server-wide popular id-set.
|
||||
type calendarPopularSource interface {
|
||||
GetRecommendationCache(ctx context.Context, userID int, profileID, recType, sourceItemID string) ([]recommendations.ScoredItem, error)
|
||||
}
|
||||
|
||||
// calendarTrendingSource reads the external-trending snapshot.
|
||||
type calendarTrendingSource interface {
|
||||
Get(ctx context.Context, source, window string) (sections.TrendingSnapshot, bool, error)
|
||||
}
|
||||
|
||||
const (
|
||||
calendarFilterAll = "all"
|
||||
calendarFilterEverything = "everything"
|
||||
calendarFilterFollowing = "following"
|
||||
calendarFilterFavorites = "favorites"
|
||||
calendarFilterWatchlist = "watchlist"
|
||||
calendarFilterPopular = "popular"
|
||||
calendarFilterTrending = "trending"
|
||||
)
|
||||
|
||||
// The Trending preset reads this canonical external-trending snapshot
|
||||
// (see internal/sections trending snapshots).
|
||||
const (
|
||||
calendarTrendingSnapshotSource = "tmdb"
|
||||
calendarTrendingSnapshotWindow = "week"
|
||||
)
|
||||
|
||||
// CalendarHandler handles the calendar endpoint.
|
||||
type CalendarHandler struct {
|
||||
repo calendarRepository
|
||||
detailSvc *catalog.DetailService
|
||||
personal calendarPersonalRepo // nil-tolerant (per-profile presets degrade to empty)
|
||||
popular calendarPopularSource // nil when recommendations disabled
|
||||
trending calendarTrendingSource // nil when trending disabled
|
||||
}
|
||||
|
||||
// NewCalendarHandler creates a new CalendarHandler.
|
||||
func NewCalendarHandler(repo *catalog.CalendarRepository, detailSvc *catalog.DetailService) *CalendarHandler {
|
||||
return &CalendarHandler{repo: repo, detailSvc: detailSvc}
|
||||
// NewCalendarHandler creates a new CalendarHandler. The repo doubles as the
|
||||
// per-profile resolver since *catalog.CalendarRepository implements both.
|
||||
func NewCalendarHandler(repo *catalog.CalendarRepository, detailSvc *catalog.DetailService, popular calendarPopularSource, trending calendarTrendingSource) *CalendarHandler {
|
||||
return &CalendarHandler{repo: repo, detailSvc: detailSvc, personal: repo, popular: popular, trending: trending}
|
||||
}
|
||||
|
||||
// --- Response types ---
|
||||
@@ -36,8 +79,12 @@ type calendarEventResponse struct {
|
||||
EpisodeNumber *int `json:"episode_number,omitempty"`
|
||||
AirDate string `json:"air_date"`
|
||||
AirTime *string `json:"air_time,omitempty"`
|
||||
AirAt *string `json:"air_at,omitempty"`
|
||||
AirTimezone *string `json:"air_timezone,omitempty"`
|
||||
LocalAirDate string `json:"local_air_date"`
|
||||
PosterURL string `json:"poster_url,omitempty"`
|
||||
PosterThumbhash string `json:"poster_thumbhash,omitempty"`
|
||||
Watched bool `json:"watched"`
|
||||
Badges []string `json:"badges"`
|
||||
}
|
||||
|
||||
@@ -75,31 +122,39 @@ func (h *CalendarHandler) HandleGetCalendar(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
filter := q.Get("filter")
|
||||
if filter == "" {
|
||||
filter = "all"
|
||||
filter = calendarFilterAll
|
||||
}
|
||||
switch filter {
|
||||
case "all", "favorites", "watchlist":
|
||||
case calendarFilterAll, calendarFilterEverything, calendarFilterFollowing,
|
||||
calendarFilterFavorites, calendarFilterWatchlist, calendarFilterPopular, calendarFilterTrending:
|
||||
default:
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "filter must be all, favorites, or watchlist")
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "invalid filter")
|
||||
return
|
||||
}
|
||||
|
||||
af := requestAccessFilter(r)
|
||||
|
||||
if (filter == "favorites" || filter == "watchlist") && af.ProfileID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "profile required for favorites/watchlist filter")
|
||||
viewerLocation := catalog.CalendarLocation(q.Get("timezone"))
|
||||
|
||||
restrict, ids, err := h.resolveCalendarRestriction(r.Context(), filter, af)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "failed to resolve calendar filter")
|
||||
return
|
||||
}
|
||||
// A restricting preset with no ids can never match — skip the windowed query.
|
||||
if restrict && len(ids) == 0 {
|
||||
writeJSON(w, http.StatusOK, calendarResponse{Events: []calendarDayResponse{}})
|
||||
return
|
||||
}
|
||||
|
||||
cf := catalog.CalendarFilter{
|
||||
Start: start,
|
||||
End: end,
|
||||
Filter: filter,
|
||||
Start: start.AddDate(0, 0, -2),
|
||||
End: end.AddDate(0, 0, 2),
|
||||
AllowedLibraryIDs: af.AllowedLibraryIDs,
|
||||
DisabledLibraryIDs: af.DisabledLibraryIDs,
|
||||
MaxContentRating: af.MaxContentRating,
|
||||
UserID: af.UserID,
|
||||
ProfileID: af.ProfileID,
|
||||
RestrictByIDs: restrict,
|
||||
RestrictToIDs: ids,
|
||||
}
|
||||
|
||||
if v := q.Get("library_id"); v != "" {
|
||||
@@ -117,12 +172,82 @@ func (h *CalendarHandler) HandleGetCalendar(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
watched := h.resolveWatched(r.Context(), af, events)
|
||||
|
||||
// Group events by date and build response.
|
||||
days := groupEventsByDate(events, r, h.detailSvc)
|
||||
days := groupEventsByDate(events, r, h.detailSvc, start, end, viewerLocation, watched)
|
||||
writeJSON(w, http.StatusOK, calendarResponse{Events: days})
|
||||
}
|
||||
|
||||
func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSvc *catalog.DetailService) []calendarDayResponse {
|
||||
// resolveCalendarRestriction maps a preset to an id-set restriction. restrict=false
|
||||
// means no restriction (everything/all). A nil/missing source degrades that preset
|
||||
// to an empty id-set (which the caller renders as an empty calendar).
|
||||
func (h *CalendarHandler) resolveCalendarRestriction(ctx context.Context, filter string, af catalog.AccessFilter) (restrict bool, ids []string, err error) {
|
||||
switch filter {
|
||||
case calendarFilterAll, calendarFilterEverything:
|
||||
return false, nil, nil
|
||||
case calendarFilterFollowing, calendarFilterFavorites, calendarFilterWatchlist:
|
||||
if h.personal == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
switch filter {
|
||||
case calendarFilterFavorites:
|
||||
ids, err = h.personal.ListFavoriteItemIDs(ctx, af.UserID, af.ProfileID)
|
||||
case calendarFilterWatchlist:
|
||||
ids, err = h.personal.ListWatchlistItemIDs(ctx, af.UserID, af.ProfileID)
|
||||
default: // following
|
||||
ids, err = h.personal.ListFollowedItemIDs(ctx, af.UserID, af.ProfileID)
|
||||
}
|
||||
return true, ids, err
|
||||
case calendarFilterPopular:
|
||||
if h.popular == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
items, err := h.popular.GetRecommendationCache(ctx, recommendations.GlobalCacheUserID, recommendations.GlobalCacheProfileID, recommendations.RecTypePopular, "")
|
||||
if err != nil {
|
||||
return true, nil, err
|
||||
}
|
||||
ids = make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
ids = append(ids, it.MediaItemID)
|
||||
}
|
||||
return true, ids, nil
|
||||
case calendarFilterTrending:
|
||||
if h.trending == nil {
|
||||
return true, nil, nil
|
||||
}
|
||||
snap, ok, err := h.trending.Get(ctx, calendarTrendingSnapshotSource, calendarTrendingSnapshotWindow)
|
||||
if err != nil {
|
||||
return true, nil, err
|
||||
}
|
||||
if !ok {
|
||||
return true, nil, nil
|
||||
}
|
||||
return true, snap.ContentIDs, nil
|
||||
default:
|
||||
return false, nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// resolveWatched decorates events with the profile's completed status. Best-effort:
|
||||
// a lookup failure logs and returns no watched marks rather than failing the page.
|
||||
func (h *CalendarHandler) resolveWatched(ctx context.Context, af catalog.AccessFilter, events []catalog.CalendarEvent) map[string]bool {
|
||||
if h.personal == nil || len(events) == 0 {
|
||||
return map[string]bool{}
|
||||
}
|
||||
ids := make([]string, 0, len(events))
|
||||
for _, ev := range events {
|
||||
ids = append(ids, ev.ContentID)
|
||||
}
|
||||
watched, err := h.personal.ListWatchedItemIDs(ctx, af.UserID, af.ProfileID, ids)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "calendar watched overlay failed", "error", err)
|
||||
return map[string]bool{}
|
||||
}
|
||||
return watched
|
||||
}
|
||||
|
||||
func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSvc *catalog.DetailService, start, end time.Time, viewerLocation *time.Location, watched map[string]bool) []calendarDayResponse {
|
||||
if len(events) == 0 {
|
||||
return []calendarDayResponse{}
|
||||
}
|
||||
@@ -144,17 +269,78 @@ func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSv
|
||||
posterURLs = detailSvc.PresignImageURLs(r.Context(), posterPaths, "poster", "small")
|
||||
}
|
||||
|
||||
type preparedCalendarEvent struct {
|
||||
event catalog.CalendarEvent
|
||||
localDate string
|
||||
sourceDate string
|
||||
localTime time.Time
|
||||
hasTime bool
|
||||
airAtString *string
|
||||
}
|
||||
|
||||
startDate := start.Format("2006-01-02")
|
||||
endDate := end.Format("2006-01-02")
|
||||
|
||||
prepared := make([]preparedCalendarEvent, 0, len(events))
|
||||
for _, ev := range events {
|
||||
localTime, hasTime := catalog.CalendarEventLocalTime(ev.AirDate, ev.AirTime, ev.AirTimezone, viewerLocation)
|
||||
localDate := localTime.Format("2006-01-02")
|
||||
if localDate < startDate || localDate > endDate {
|
||||
continue
|
||||
}
|
||||
var airAtString *string
|
||||
if airAt := catalog.CalendarEventAirAt(ev.AirDate, ev.AirTime, ev.AirTimezone); airAt != nil {
|
||||
formatted := airAt.Format(time.RFC3339)
|
||||
airAtString = &formatted
|
||||
}
|
||||
prepared = append(prepared, preparedCalendarEvent{
|
||||
event: ev,
|
||||
localDate: localDate,
|
||||
sourceDate: ev.AirDate.Format("2006-01-02"),
|
||||
localTime: localTime,
|
||||
hasTime: hasTime,
|
||||
airAtString: airAtString,
|
||||
})
|
||||
}
|
||||
if len(prepared) == 0 {
|
||||
return []calendarDayResponse{}
|
||||
}
|
||||
|
||||
// Order each local day by the wall-clock time the viewer actually sees,
|
||||
// then place date-only entries (no air_time) after timed entries.
|
||||
sort.SliceStable(prepared, func(i, j int) bool {
|
||||
left, right := prepared[i], prepared[j]
|
||||
if left.localDate != right.localDate {
|
||||
return left.localDate < right.localDate
|
||||
}
|
||||
if left.hasTime != right.hasTime {
|
||||
return left.hasTime
|
||||
}
|
||||
if left.hasTime && !left.localTime.Equal(right.localTime) {
|
||||
return left.localTime.Before(right.localTime)
|
||||
}
|
||||
if left.event.Title != right.event.Title {
|
||||
return left.event.Title < right.event.Title
|
||||
}
|
||||
if left.event.SeasonNumber != nil && right.event.SeasonNumber != nil && *left.event.SeasonNumber != *right.event.SeasonNumber {
|
||||
return *left.event.SeasonNumber < *right.event.SeasonNumber
|
||||
}
|
||||
if left.event.EpisodeNumber != nil && right.event.EpisodeNumber != nil && *left.event.EpisodeNumber != *right.event.EpisodeNumber {
|
||||
return *left.event.EpisodeNumber < *right.event.EpisodeNumber
|
||||
}
|
||||
return left.event.ContentID < right.event.ContentID
|
||||
})
|
||||
|
||||
var days []calendarDayResponse
|
||||
var currentDay *calendarDayResponse
|
||||
|
||||
for _, ev := range events {
|
||||
dateStr := ev.AirDate.Format("2006-01-02")
|
||||
|
||||
if currentDay == nil || currentDay.Date != dateStr {
|
||||
for _, item := range prepared {
|
||||
ev := item.event
|
||||
if currentDay == nil || currentDay.Date != item.localDate {
|
||||
if currentDay != nil {
|
||||
days = append(days, *currentDay)
|
||||
}
|
||||
currentDay = &calendarDayResponse{Date: dateStr}
|
||||
currentDay = &calendarDayResponse{Date: item.localDate}
|
||||
}
|
||||
|
||||
badges := buildBadges(ev)
|
||||
@@ -167,10 +353,14 @@ func groupEventsByDate(events []catalog.CalendarEvent, r *http.Request, detailSv
|
||||
SeriesID: ev.SeriesID,
|
||||
SeasonNumber: ev.SeasonNumber,
|
||||
EpisodeNumber: ev.EpisodeNumber,
|
||||
AirDate: dateStr,
|
||||
AirDate: item.sourceDate,
|
||||
AirTime: ev.AirTime,
|
||||
AirAt: item.airAtString,
|
||||
AirTimezone: ev.AirTimezone,
|
||||
LocalAirDate: item.localDate,
|
||||
PosterURL: posterURLs[ev.PosterPath],
|
||||
PosterThumbhash: ev.PosterThumbhash,
|
||||
Watched: watched[ev.ContentID],
|
||||
Badges: badges,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
)
|
||||
|
||||
type stubCalendarRepo struct {
|
||||
@@ -110,6 +112,62 @@ func TestHandleGetCalendar_ReturnsEmptyEvents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_ExpandsRepoRangeAndGroupsByViewerLocalDate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
airTime := "00:30"
|
||||
airTimezone := "Asia/Tokyo"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "episode-1",
|
||||
Type: "episode",
|
||||
Title: "Series",
|
||||
SeriesID: ptrString("series-1"),
|
||||
AirDate: time.Date(2026, time.January, 2, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &airTime,
|
||||
AirTimezone: &airTimezone,
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if got := repo.last.Start.Format("2006-01-02"); got != "2025-12-30" {
|
||||
t.Fatalf("repo start = %s, want expanded 2025-12-30", got)
|
||||
}
|
||||
if got := repo.last.End.Format("2006-01-02"); got != "2026-01-03" {
|
||||
t.Fatalf("repo end = %s, want expanded 2026-01-03", got)
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
if resp.Events[0].Date != "2026-01-01" {
|
||||
t.Fatalf("day = %q, want viewer-local 2026-01-01", resp.Events[0].Date)
|
||||
}
|
||||
item := resp.Events[0].Items[0]
|
||||
if item.AirDate != "2026-01-02" {
|
||||
t.Fatalf("air_date = %q, want source date 2026-01-02", item.AirDate)
|
||||
}
|
||||
if item.LocalAirDate != "2026-01-01" {
|
||||
t.Fatalf("local_air_date = %q, want 2026-01-01", item.LocalAirDate)
|
||||
}
|
||||
if item.AirAt == nil || *item.AirAt != "2026-01-01T15:30:00Z" {
|
||||
t.Fatalf("air_at = %v, want 2026-01-01T15:30:00Z", item.AirAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_GroupsEventsAndBatchResolvesCardPosters(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -189,6 +247,176 @@ func TestHandleGetCalendar_GroupsEventsAndBatchResolvesCardPosters(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_OrdersEventsWithoutTimezoneByWallClock(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
late := "22:00"
|
||||
early := "10:00"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "ep-late",
|
||||
Type: "episode",
|
||||
Title: "Aaa Show", // alphabetically first, but airs late
|
||||
SeriesID: ptrString("series-late"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &late,
|
||||
// no AirTimezone -> air_at is nil
|
||||
},
|
||||
{
|
||||
ContentID: "ep-early",
|
||||
Type: "episode",
|
||||
Title: "Zzz Show", // alphabetically last, but airs early
|
||||
SeriesID: ptrString("series-early"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &early,
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
items := resp.Events[0].Items
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items len = %d, want 2", len(items))
|
||||
}
|
||||
if items[0].ContentID != "ep-early" || items[1].ContentID != "ep-late" {
|
||||
t.Fatalf("order = [%s, %s], want [ep-early, ep-late] (ascending by air_time)",
|
||||
items[0].ContentID, items[1].ContentID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_InterleavesZonedAndUnzonedByViewerLocalTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
eightPM := "20:00"
|
||||
sixPM := "18:00"
|
||||
londonNine := "21:00"
|
||||
london := "Europe/London"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "alpha-8pm",
|
||||
Type: "episode",
|
||||
Title: "Alpha", // would sort first alphabetically
|
||||
SeriesID: ptrString("series-alpha"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &eightPM, // displays 8:00 PM ET
|
||||
},
|
||||
{
|
||||
ContentID: "bravo-4pm",
|
||||
Type: "episode",
|
||||
Title: "Bravo",
|
||||
SeriesID: ptrString("series-bravo"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &londonNine, // 21:00 GMT -> 16:00 ET (4:00 PM)
|
||||
AirTimezone: &london,
|
||||
},
|
||||
{
|
||||
ContentID: "charlie-6pm",
|
||||
Type: "episode",
|
||||
Title: "Charlie",
|
||||
SeriesID: ptrString("series-charlie"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &sixPM, // displays 6:00 PM ET
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
got := make([]string, 0, len(resp.Events[0].Items))
|
||||
for _, item := range resp.Events[0].Items {
|
||||
got = append(got, item.ContentID)
|
||||
}
|
||||
want := []string{"bravo-4pm", "charlie-6pm", "alpha-8pm"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("items = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("order = %v, want %v (ascending by viewer-local time)", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_DateOnlyEntriesSortAfterTimedEntries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
noon := "12:00"
|
||||
repo := &stubCalendarRepo{
|
||||
events: []catalog.CalendarEvent{
|
||||
{
|
||||
ContentID: "movie-1",
|
||||
Type: "movie",
|
||||
Title: "Aaa Movie", // alphabetically first, but has no air_time
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
ContentID: "ep-noon",
|
||||
Type: "episode",
|
||||
Title: "Zzz Show",
|
||||
SeriesID: ptrString("series-z"),
|
||||
AirDate: time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
AirTime: &noon,
|
||||
},
|
||||
},
|
||||
}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-01-01&end=2026-01-01&timezone=America/New_York", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(resp.Events) != 1 {
|
||||
t.Fatalf("days len = %d, want 1", len(resp.Events))
|
||||
}
|
||||
items := resp.Events[0].Items
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items len = %d, want 2", len(items))
|
||||
}
|
||||
if items[0].ContentID != "ep-noon" || items[1].ContentID != "movie-1" {
|
||||
t.Fatalf("order = [%s, %s], want [ep-noon, movie-1] (timed before date-only)",
|
||||
items[0].ContentID, items[1].ContentID)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrString(value string) *string {
|
||||
return &value
|
||||
}
|
||||
@@ -196,3 +424,162 @@ func ptrString(value string) *string {
|
||||
func ptrInt(value int) *int {
|
||||
return &value
|
||||
}
|
||||
|
||||
type stubCalendarPersonal struct {
|
||||
followed []string
|
||||
favorites []string
|
||||
watchlist []string
|
||||
watched map[string]bool
|
||||
lastWatchedIDs []string
|
||||
}
|
||||
|
||||
func (s *stubCalendarPersonal) ListFollowedItemIDs(_ context.Context, _ int, _ string) ([]string, error) {
|
||||
return s.followed, nil
|
||||
}
|
||||
func (s *stubCalendarPersonal) ListFavoriteItemIDs(_ context.Context, _ int, _ string) ([]string, error) {
|
||||
return s.favorites, nil
|
||||
}
|
||||
func (s *stubCalendarPersonal) ListWatchlistItemIDs(_ context.Context, _ int, _ string) ([]string, error) {
|
||||
return s.watchlist, nil
|
||||
}
|
||||
func (s *stubCalendarPersonal) ListWatchedItemIDs(_ context.Context, _ int, _ string, ids []string) (map[string]bool, error) {
|
||||
s.lastWatchedIDs = ids
|
||||
return s.watched, nil
|
||||
}
|
||||
|
||||
type stubPopularSource struct{ items []recommendations.ScoredItem }
|
||||
|
||||
func (s *stubPopularSource) GetRecommendationCache(_ context.Context, _ int, _, _, _ string) ([]recommendations.ScoredItem, error) {
|
||||
return s.items, nil
|
||||
}
|
||||
|
||||
type stubTrendingSource struct {
|
||||
snap sections.TrendingSnapshot
|
||||
ok bool
|
||||
}
|
||||
|
||||
func (s *stubTrendingSource) Get(_ context.Context, _, _ string) (sections.TrendingSnapshot, bool, error) {
|
||||
return s.snap, s.ok, nil
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_FollowingPassesFollowedIDs(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo, personal: &stubCalendarPersonal{followed: []string{"s1", "s2"}}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=following", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if !repo.last.RestrictByIDs {
|
||||
t.Fatalf("expected RestrictByIDs true")
|
||||
}
|
||||
if got := repo.last.RestrictToIDs; len(got) != 2 || got[0] != "s1" || got[1] != "s2" {
|
||||
t.Fatalf("RestrictToIDs = %v, want [s1 s2]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_PopularReadsCache(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{
|
||||
repo: repo,
|
||||
popular: &stubPopularSource{items: []recommendations.ScoredItem{{MediaItemID: "p1"}, {MediaItemID: "p2"}}},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=popular", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if !repo.last.RestrictByIDs || len(repo.last.RestrictToIDs) != 2 || repo.last.RestrictToIDs[0] != "p1" {
|
||||
t.Fatalf("RestrictToIDs = %v, want [p1 p2]", repo.last.RestrictToIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_TrendingReadsSnapshot(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{
|
||||
repo: repo,
|
||||
trending: &stubTrendingSource{ok: true, snap: sections.TrendingSnapshot{ContentIDs: []string{"t1"}}},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=trending", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if !repo.last.RestrictByIDs || len(repo.last.RestrictToIDs) != 1 || repo.last.RestrictToIDs[0] != "t1" {
|
||||
t.Fatalf("RestrictToIDs = %v, want [t1]", repo.last.RestrictToIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_EverythingHasNoRestriction(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=everything", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if repo.last.RestrictByIDs {
|
||||
t.Fatalf("expected no restriction for everything")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_RejectsUnknownFilter(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=bogus", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("expected repo not called, got %d", repo.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_EmptyFollowedShortCircuits(t *testing.T) {
|
||||
repo := &stubCalendarRepo{}
|
||||
handler := &CalendarHandler{repo: repo, personal: &stubCalendarPersonal{followed: nil}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=following", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("expected ListEvents skipped for empty followed set, got %d calls", repo.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGetCalendar_MarksWatchedItems(t *testing.T) {
|
||||
repo := &stubCalendarRepo{events: []catalog.CalendarEvent{
|
||||
{ContentID: "ep-1", Type: "episode", Title: "Show", AirDate: time.Date(2026, time.April, 8, 0, 0, 0, 0, time.UTC)},
|
||||
{ContentID: "ep-2", Type: "episode", Title: "Show", AirDate: time.Date(2026, time.April, 8, 0, 0, 0, 0, time.UTC)},
|
||||
}}
|
||||
handler := &CalendarHandler{repo: repo, personal: &stubCalendarPersonal{watched: map[string]bool{"ep-1": true}}}
|
||||
req := httptest.NewRequest(http.MethodGet, "/calendar?start=2026-04-06&end=2026-04-12&filter=everything", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.HandleGetCalendar(rec, req)
|
||||
|
||||
var resp calendarResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
watched := map[string]bool{}
|
||||
for _, day := range resp.Events {
|
||||
for _, item := range day.Items {
|
||||
watched[item.ContentID] = item.Watched
|
||||
}
|
||||
}
|
||||
if !watched["ep-1"] || watched["ep-2"] {
|
||||
t.Fatalf("watched flags = %v, want ep-1 true / ep-2 false", watched)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
@@ -80,6 +81,18 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request
|
||||
overlaySummaries := h.itemsH.listOverlaySummaries(r.Context(), result.Items, h.itemsH.accessFilter(r))
|
||||
userStates := h.itemsH.listItemUserStates(r, result.Items)
|
||||
episodeMetadata := h.itemsH.listEpisodeBrowseMetadata(r.Context(), result.Items)
|
||||
sortField := catalog.NormalizeQuerySort(req.Query.Sort).Field
|
||||
store, profileID, _ := h.itemsH.userStoreForRequest(r)
|
||||
sortMetrics := h.itemsH.listSortMetrics(
|
||||
r.Context(),
|
||||
result.Items,
|
||||
sortField,
|
||||
h.itemsH.accessFilter(r),
|
||||
overlaySummaries,
|
||||
store,
|
||||
apimw.GetUserID(r.Context()),
|
||||
profileID,
|
||||
)
|
||||
items := make([]itemListResponse, 0, len(result.Items))
|
||||
for _, item := range result.Items {
|
||||
resp := h.itemsH.toItemListResponseWithOverlay(
|
||||
@@ -93,6 +106,7 @@ func (h *CatalogHandler) HandleGetCatalog(w http.ResponseWriter, r *http.Request
|
||||
resp.SeasonNumber = meta.SeasonNumber
|
||||
resp.EpisodeNumber = meta.EpisodeNumber
|
||||
}
|
||||
resp.SortMetrics = sortMetrics[item.ContentID]
|
||||
items = append(items, resp)
|
||||
}
|
||||
|
||||
@@ -209,6 +223,9 @@ func (h *CatalogHandler) HandleGetCatalogFacetSearch(w http.ResponseWriter, r *h
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "limit must be a positive integer")
|
||||
return
|
||||
}
|
||||
if n > 100 {
|
||||
n = 100
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user