diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 83bffaa6..c5c76e5e 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -2052,6 +2052,7 @@ func main() { intervalMs = int64(settings.DefaultPollIntervalSeconds) * 1000 } taskMgr.Register(tasks.NewAutoscanPollTask(autoscanSvc, intervalMs)) + taskMgr.Register(tasks.NewAutoscanWebhookRetryTask(autoscanSvc)) } reconcileProviderIDRepo := catalog.NewProviderIDRepository(deps.DB) reconcileEpisodeRepo := catalog.NewEpisodeRepository(deps.DB) diff --git a/docs/superpowers/plans/2026-07-09-autoscan-arr-webhook-intake-plan.md b/docs/superpowers/plans/2026-07-09-autoscan-arr-webhook-intake-plan.md new file mode 100644 index 00000000..d8844975 --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-autoscan-arr-webhook-intake-plan.md @@ -0,0 +1,376 @@ +# Implementation Plan: Autoscan Sonarr/Radarr Webhook Intake + +**Spec:** `docs/superpowers/specs/2026-07-09-autoscan-arr-webhook-intake-design.md` +(Option A, decisions locked — read the spec's "Resolved review decisions" before +starting.) +**Date:** 2026-07-09 + +Commands assume the repository root is the cwd. + +## Overview + +Add a webhook delivery mode to Autoscan so Sonarr/Radarr can POST import, +upgrade, rename, and delete notifications directly to Silo without an arr API +key. Webhook sources bind to a host-discovered built-in identity +(`silo.autoscan.arr-webhook`), never the installed ARR plugin. The host parses +payloads and feeds the existing rewrite → resolve → suppress → enqueue → event +pipeline. + +Non-negotiable design points from the review: + +1. **Built-in source identity** — no plugin install required for webhook mode. +2. **Webhook deliveries are never single-flight-dropped.** Only poll cycles may + be skipped when a running event exists (their marker window is re-read). +3. Event rows record `delivery_mode` and `provider_event_type` from day one. +4. Unknown arr event types → `202` no-op. Secret lookup uses plain SHA-256. + Webhook URL stays redisplayable on admin endpoints. + +Suggested commit sequence is one commit per phase below, all on one branch, PR +subject `feat(autoscan): add Sonarr/Radarr webhook intake`. + +## Phase 1 — Migration + +Create with `make migrate-create NAME=autoscan_webhook_intake` (single +timestamped Goose migration; do not hand-number, do not create paired +up/down files). + +Up: + +```sql +ALTER TABLE autoscan_sources + ADD COLUMN delivery_mode text NOT NULL DEFAULT 'poll' + CONSTRAINT autoscan_sources_delivery_mode_check + CHECK (delivery_mode = ANY (ARRAY['poll'::text, 'webhook'::text])); + +CREATE TABLE autoscan_webhook_endpoints ( + source_id uuid PRIMARY KEY + REFERENCES autoscan_sources(id) ON DELETE CASCADE, + secret_hash text UNIQUE NOT NULL, + secret_ref text NOT NULL, + secret_suffix text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + rotated_at timestamptz, + last_received_at timestamptz, + last_error_at timestamptz, + last_error_message text NOT NULL DEFAULT '' +); + +ALTER TABLE autoscan_events + ADD COLUMN delivery_mode text NOT NULL DEFAULT 'poll', + ADD COLUMN provider_event_type text NOT NULL DEFAULT ''; +``` + +Down reverses all three (drop table, drop columns). Verify with +`make migrate-status` / `make migrate-up` against a scratch DB, and check +rollback. + +## Phase 2 — Built-in source identity + +**`internal/autoscan/discovery.go`** + +- Add exported constants: + - `BuiltinArrWebhookPluginID = "silo.autoscan.arr-webhook"` + - `BuiltinArrWebhookCapabilityID = "arr-webhook"` + - display name `"Sonarr/Radarr Webhook"`. +- Add a composite lister so the built-in identity lives beside the discovery + types it extends (not in wiring code): + + ```go + // WithBuiltinSources wraps a ScanSourceLister and appends host-built-in + // source identities that need no plugin installation. + func WithBuiltinSources(inner ScanSourceLister, builtins ...DiscoveredSource) ScanSourceLister + ``` + + It must tolerate `inner == nil` (return only builtins) because + `Service.lister` is already allowed to be nil. + +**`internal/api/autoscan_wiring.go`** + +- In `BuildAutoscanService`, wrap the existing lister: + + ```go + autoscan.WithBuiltinSources( + PluginScanSourceLister{installationStore}, + autoscan.BuiltinArrWebhookSource(), // helper returning the DiscoveredSource + ) + ``` + +Tests (`internal/autoscan/discovery_test.go`): builtin appears with and without +an inner lister; plugin entries pass through unchanged. + +## Phase 3 — Repository and types + +**`internal/autoscan/types.go`** + +- `Source`: add `DeliveryMode string` (values `"poll"`/`"webhook"`; constants + `DeliveryModePoll`, `DeliveryModeWebhook`). +- `Event`: add `DeliveryMode string`, `ProviderEventType string`. +- `EventCreate`: add `DeliveryMode string`, `ProviderEventType string`, + `SkipRunningCheck bool`. +- New: + + ```go + type WebhookEndpoint struct { + SourceID string + SecretSuffix string + CreatedAt time.Time + RotatedAt *time.Time + LastReceivedAt *time.Time + LastErrorAt *time.Time + LastErrorMessage string + } + ``` + + The plaintext token and hash never live on the struct that flows to + handlers/responses; creation/rotation return the token separately. + +**`internal/autoscan/repository.go`** + +- Thread `delivery_mode` through `CreateSource`, `UpdateSource`, + `ListSources`, `ListEnabledSources`, `GetSource` (default `'poll'` when + empty on write). +- `CreateEvent`: write the two new columns. When `in.SkipRunningCheck` is + true, use the simple no-transaction insert path (the same shape as the + existing `sourceID == ""` branch, but with the source id set) — no advisory + lock, no running-row check. Poll-mode behavior is unchanged. +- `FinishEvent` / `ListEvents` / `ListRunningEvents` / scan-event joins: + select and populate the new event fields. +- Webhook endpoint methods (token = 32 bytes from `crypto/rand`, + `base64.RawURLEncoding`; hash = hex SHA-256 of the token; `secret_ref` = + `r.cipher.Encrypt(token, secret.RowAAD("autoscan_webhook_endpoints", "secret_ref", sourceID))`; + suffix = last 6 chars of the token): + + ```go + // CreateWebhookEndpoint creates (or returns the existing) endpoint; token is + // "" when the endpoint already existed. + CreateWebhookEndpoint(ctx, sourceID string) (WebhookEndpoint, token string, err error) + RotateWebhookEndpoint(ctx, sourceID string) (WebhookEndpoint, token string, err error) + DeleteWebhookEndpoint(ctx, sourceID string) error + GetWebhookEndpoint(ctx, sourceID string) (WebhookEndpoint, error) + // RevealWebhookToken decrypts secret_ref for admin redisplay. + RevealWebhookToken(ctx, sourceID string) (string, error) + // ResolveWebhookToken maps a raw delivery token to its source. Constant + // shape: hash the token, look up by secret_hash, join autoscan_sources. + // Returns ErrNotFound for unknown hashes. + ResolveWebhookToken(ctx, token string) (Source, WebhookEndpoint, error) + TouchWebhookReceived(ctx, sourceID string) error + RecordWebhookError(ctx, sourceID, msg string) error // bounded, sanitized by caller + ``` + +DB-backed tests follow the existing `SILO_TEST_DATABASE_URL` pattern used by +the rest of `internal/autoscan`. Cover: create/rotate invalidates the old +token, resolve-by-token round-trip, cascade delete with the source, reveal +decrypts to the original token. + +## Phase 4 — Service: shared consume path + `IngestChanges` + +**`internal/autoscan/service.go`** + +- Extract everything in `PollOnce` after `PollChanges` returns into: + + ```go + type consumeOptions struct { + EventID int64 + Marker string // poll only; "" for webhook + NextMarker string // poll only + AdvanceMarker bool // false for webhook + } + + func (s *Service) consumeSourceChanges(ctx context.Context, src Source, changes []Change, opts consumeOptions) error + ``` + + It owns: `rewriteChanges` → `resolveAndClaim` → target-cap collapse → + `enqueueScanTargets` (+ `releaseClaims` on failure) → the + transient-error / unresolved / success status decision → `finishEvent`. + Marker advancement stays conditional on `opts.AdvanceMarker`; the hold-marker + comments and semantics in `PollOnce` must survive the refactor intact. +- `PollOnce`: skip sources with `src.DeliveryMode == DeliveryModeWebhook` + (before the interval check), then delegate to `consumeSourceChanges`. + Existing poll tests in `service_test.go` must pass unmodified except for + construction changes. +- Add: + + ```go + type ChangeIngest struct { + SourceID string + ProviderEventType string + Changes []Change + ReceivedAt time.Time + } + + type IngestResult struct { + Enqueued int + Suppressed int + Unresolved bool + } + + func (s *Service) IngestChanges(ctx context.Context, in ChangeIngest) (IngestResult, error) + ``` + + Behavior: + - Load the source; reject non-webhook sources. + - `CreateEvent` with `DeliveryMode: "webhook"`, `ProviderEventType`, + `SkipRunningCheck: true` — deliveries are never dropped because another + event is running (spec: "Security and reliability"). + - Delegate to `consumeSourceChanges` with `AdvanceMarker: false` and empty + markers. + - Transient resolve failure → finish event as error and return an error (the + handler maps it; a duplicate later delivery is safe). All paths outside + Silo folders → finish as `unresolved`, return success. + +Service tests: webhook ingest enqueues the same targets as an equivalent poll; +concurrent ingests for one source both complete (no `ErrPollAlreadyRunning`); +suppressor claims dedupe overlapping targets; `PollOnce` skips webhook sources. + +## Phase 5 — Parser: `internal/autoscan/arrwebhook` + +New package, no dependencies on api/handlers. + +```go +type ParsedWebhook struct { + Provider string // "sonarr" | "radarr" + EventType string + Test bool + Changes []autoscan.Change +} + +func Parse(provider string, body []byte) (ParsedWebhook, error) +``` + +- `provider` is `sonarr`, `radarr`, or `auto`; `auto` infers from top-level + `series` vs `movie` keys. Inference failure on a non-test event returns a + typed error the handler maps to 400 (sanitized message, no payload echo). +- Event handling per the spec's "Payload parsing" section: Test → no changes; + Download/Import/Upgrade/DownloadComplete → imported file paths; Rename → new + **and** previous paths; EpisodeFileDelete/MovieFileDelete → deleted path. + All paths emit `ChangeScopeFile` (`resolveChange` already falls back through + `ResolveVanishedPath` for vanished files). Series/movie folder paths are a + subtree fallback (`ChangeScopeSubtree`) only when no file path exists. +- Unknown event types return `ParsedWebhook` with no changes and a marker the + handler can distinguish (e.g. `EventType` set, `Changes` empty, no error) — + they must not be an error. +- Dedupe exact paths before returning. + +Fixtures in `internal/autoscan/arrwebhook/testdata/`: Sonarr Download, Rename, +EpisodeFileDelete, Test; Radarr Download, Rename, MovieFileDelete, Test; plus +an unknown-event and a malformed body. Build them from the Servarr custom- +scripts docs linked in the spec. + +## Phase 6 — HTTP: public delivery route + admin endpoints + +**New file `internal/api/handlers/autoscan_webhook.go`** + +Public handler `HandleWebhookDelivery` (`POST /api/v1/autoscan/webhooks/{token}`): + +1. `http.MaxBytesReader` at 256 KiB → 413 on overflow. +2. Hash token, `ResolveWebhookToken` → 404 on miss (identical body for + deleted/never-existed; do not distinguish). +3. Global autoscan disabled or source disabled → 202, no enqueue, still + `TouchWebhookReceived`. +4. `arrwebhook.Parse` with `source_config.webhook_provider` (default `auto`) + → 400 on parse error; 202 no-op for Test and unknown event types. +5. `IngestChanges`; on success `TouchWebhookReceived` and return `202`. On + transient ingest failure, `RecordWebhookError` (bounded message) and return + `500` so arr retries — a duplicate later delivery is safe. Log only source + id, provider, event type, and path count. **Never log the token, URL, or + body.** + +Admin handlers (same file, wired through `AutoscanHandler` deps): + +- `POST /admin/autoscan/sources/{id}/webhook` → create-if-missing, return + URL + status. +- `POST /admin/autoscan/sources/{id}/webhook/rotate` → rotate, return new URL. +- `DELETE /admin/autoscan/sources/{id}/webhook` → delete. +- URL construction: `deps.PublicURL + "/api/v1/autoscan/webhooks/" + token` + when `PublicURL` is set; otherwise return the relative path and let the UI + prepend `window.location.origin`. Response field is `webhook_url` either way. + +**`internal/api/handlers/autoscan.go`** + +- `sourceResponse`: add `delivery_mode`, `webhook_configured`, `webhook_url`, + `webhook_secret_suffix`, `webhook_last_received_at`, `webhook_last_error_at`, + `webhook_last_error_message` (fetch endpoint rows for listed sources; a + single batched query, not N+1). +- `HandleCreateSource` / `HandleUpdateSource` validation: + - `delivery_mode` defaults to `poll`; must be `poll` or `webhook`. + - `webhook` is valid only for the built-in identity; `poll` is invalid for + the built-in identity. + - Webhook mode requires no connection; a bound connection is allowed + (rewrite suggestions only). + - `source_config.webhook_provider` ∈ {`sonarr`, `radarr`, `auto`} when set. + - The installed-capability check passes for the built-in identity + automatically because the composite lister supplies it — verify with a + handler test rather than special-casing `scanSourceInstalled`. +- Event list responses: add `delivery_mode` and `provider_event_type`. + +**`internal/api/router.go`** + +- Admin routes: add the three webhook-management routes next to the existing + `/autoscan/sources` block. +- Public route: register `r.Post("/autoscan/webhooks/{token}", ...)` alongside + the other public routes (near the auth/discord-callback blocks), gated on + `autoscanHandler != nil`. +- Rate limiting: reuse `internal/ratelimit` — mirror the + `Middleware.AuthEndpointHandler(endpoint)` per-IP pattern with a + `"autoscan_webhook"` endpoint entry (config default on the generous side, + e.g. 60/min burst 30; arr bursts are legitimate). Per-token limiting beyond + that is unnecessary for v1 given the suppressor already collapses duplicate + work; note this in the PR. + +Handler tests (`internal/api/handlers`): status-code matrix from the spec's +Testing section — 202 (ok / test / disabled / unknown event), 400, 404, 405 +(chi handles), 413, 429; rotation invalidates the old token; no token/body in +logs (assert via a captured slog handler). + +## Phase 7 — Frontend + +- `web/src/api/types.ts`: extend `AutoscanSource` with the new response + fields; add webhook endpoint response type. +- `web/src/hooks/queries/useAutoscan.ts` (+ `keys.ts`): mutations for webhook + create/rotate/delete; invalidate the sources query on success. +- `web/src/pages/admin/autoscan/SourcesPanel.tsx`: + - Delivery mode segmented control (`Poll` / `Webhook`) in the source + create/edit form. Only offered when the picked capability is the built-in + webhook identity (which forces webhook mode) or a plugin capability + (which forces poll mode) — in practice the mode follows the picker choice; + render the control as the mode indicator rather than a free toggle. + - Webhook mode panel: provider select (`Auto`/`Sonarr`/`Radarr` → + `source_config.webhook_provider`), webhook URL display with copy button + (prepend `window.location.origin` when the API returns a relative path), + rotate action with confirm, last-received / last-error status line. + - Connection picker hidden in webhook mode (or shown as explicitly optional + for rewrite suggestions); poll-interval input hidden. + - Path-rewrite editor stays visible in both modes. +- `web/src/pages/admin/autoscan/ActivityPanel.tsx`: show delivery mode and + provider event type on event rows. +- Follow existing panel patterns; 2-space/double-quote Prettier config. + +## Phase 8 — Verification + +```bash +GOWORK=off go test ./internal/autoscan/... ./internal/api/handlers +GOWORK=off go test ./... +cd web && pnpm run lint && pnpm run format:check && pnpm run build +make lint +make verify-local-paths +``` + +Manual smoke (dev): create a webhook source, paste the URL into a Sonarr +Connect → Webhook (On Import + On Rename + On File Delete), use Sonarr's +"Test" button (expect 202 and `last_received_at` updating), import a file, +confirm a webhook event row and a file-scope scan appear in the Activity +panel. + +## Risks / notes + +- The `PollOnce` refactor is the riskiest step: marker-advancement semantics + are subtle and documented inline — preserve the decision comments and rely + on the existing `service_test.go` suite as the regression harness before + adding webhook paths. +- `secret_ref` uses `internal/secret.Cipher` with row-bound AAD; never rename + the table/column in SQL later without a `db_loader`-style fallback + (encryption is AAD-bound). +- Keep `/api/v1` additive: every API change above is a new field, new + endpoint, or new route; nothing existing is renamed or repurposed. +- Client repos: none required for v1 — this is an admin-web-only surface; the + public endpoint is consumed by arr, not by Silo clients. diff --git a/docs/superpowers/specs/2026-07-09-autoscan-arr-webhook-intake-design.md b/docs/superpowers/specs/2026-07-09-autoscan-arr-webhook-intake-design.md new file mode 100644 index 00000000..121d9ed6 --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-autoscan-arr-webhook-intake-design.md @@ -0,0 +1,661 @@ +# Autoscan Sonarr/Radarr Webhook Intake + +**Status:** Reviewed 2026-07-09 — decisions locked (see "Resolved review decisions") +**Date:** 2026-07-09 +**Area:** `internal/autoscan`, `internal/api`, `web/src/pages/AdminAutoscan.tsx`, `web/src/pages/admin/autoscan/`, `migrations/sql/` + +Commands assume the repository root is the cwd. + +## Summary + +Silo's current Sonarr/Radarr Autoscan integration is pull-based. The installed +`silo.autoscan.arr` plugin implements `scan_source.v1`; the host polls it on an +interval, passes a resolved `{base_url, api_key}` connection, the plugin reads +arr `/api/v3/history`, and the host applies path rewrites, resolves scan +targets, suppresses duplicates, enqueues scans, and records poll events. + +That design works, but it means a Sonarr/Radarr Autoscan source needs an arr API +key. This spec adds a **webhook delivery mode** so Sonarr/Radarr can POST import, +upgrade, rename, and delete notifications directly to Silo. Webhook mode should +not require an arr connection or API key. The host still owns all security, +path-rewrite, scan-resolution, scan-queue, suppression, and event-bookkeeping +behavior. + +The recommended v1 is **host-native webhook intake attached to Autoscan sources**, +using a **host-discovered built-in source identity** (`silo.autoscan.arr-webhook`) +so webhook sources require neither an arr API key nor an installed plugin. +No SDK change is required for v1. The existing pull plugin remains available for +polling mode and for operators who prefer API-based history polling. + +## Current state + +- `autoscan_sources` binds a source to `plugin_id`, `capability_id`, an optional + `connection_id`, `enabled`, optional `poll_interval_seconds`, `path_rewrites`, + `source_config`, label, marker, last run time, and last error. +- `autoscan_connections` can store own credentials or a soft link to a Requests + integration. Keys are encrypted by `internal/secret`. +- `autoscan.Service.PollOnce` loads enabled sources, optionally resolves their + connection, calls the plugin `PollChanges`, applies rewrites, resolves scan + targets, suppresses duplicate work, enqueues scans, advances the marker on + success, and writes `autoscan_events`. +- `scan_source.v1` is pull only. The SDK has no "plugin received a webhook; + submit these paths to the host" RPC. +- Plugin HTTP routes exist and can be public, but they route by installation id + and are not currently connected to Autoscan enqueueing. +- The current ARR plugin has no config. It reads connection data only from + `PollChangesRequest.connection` and returns raw arr-side paths. + +## Goals + +- Let operators configure Sonarr/Radarr webhooks without storing an arr API key + in Silo. +- Keep polling mode intact and compatible with existing Autoscan sources. +- Reuse the existing host path rewrite, resolution, suppression, enqueue, and + event-history behavior for both poll and webhook deliveries. +- Give each webhook source a stable public callback URL with a rotatable secret. +- Support Sonarr/Radarr import, upgrade, rename, file-delete, and test events. +- Keep `/api/v1` additive: new fields and endpoints only. +- Avoid leaking webhook secrets or raw payloads in logs, events, or frontend + responses beyond admin-only setup surfaces. + +## Non-goals + +- No automatic creation of Sonarr/Radarr webhook connections inside arr. That + would require the arr API key this feature is trying to avoid. +- No SDK change for v1. +- No plugin-to-host push RPC for v1. +- No broad webhook framework for every future provider. This spec only defines + Sonarr/Radarr webhook intake. +- No webhook signing requirement from Sonarr/Radarr beyond the generated Silo + URL secret. If a future arr version supports request signing, add it + additively. +- No replacement of the poll plugin. Polling remains useful for fallback, + history recovery, and operators who prefer not to expose a public callback. + +## Approach comparison + +### Option A: Host-native webhook mode on Autoscan sources (recommended) + +Silo adds a public secret-bearing endpoint such as +`POST /api/v1/autoscan/webhooks/{token}`. The endpoint resolves the token to an +Autoscan source, parses the Sonarr/Radarr payload in the host, turns it into +`autoscan.Change` values, and calls a new host-side `IngestChanges` path that +shares the same rewrite/resolve/suppress/enqueue/event logic as `PollOnce`. + +**Pros** + +- No SDK release or plugin release needed for v1. +- No arr API key needed for webhook-only sources. +- Stable callback URL is source-owned, not plugin-installation-id-owned. +- Host has direct access to scan queue, resolver, suppressor, DB, event logs, + secret encryption, and rate limiting. +- Existing Autoscan activity UI and source rows remain the operator surface. +- The security boundary is simpler: one host-owned bearer secret, one host-owned + parser, one host-owned enqueue path. + +**Cons** + +- Sonarr/Radarr payload parsing becomes host code, so provider-specific logic is + no longer purely in plugins. +- Future non-arr webhook providers would need host work unless a later generic + push contract is added. + +A variant of Option A would reuse the `silo.autoscan.arr` plugin identity for +webhook sources. That was considered and rejected: `HandleCreateSource` hard +rejects any source whose `(plugin_id, capability_id)` is not a +currently-installed capability, so plugin-bound identity would force operators +to install a plugin the webhook path never calls — quietly defeating the +feature's own pitch. The built-in identity below avoids that at near-zero cost +(see "Recommended v1"). + +### Option B: SDK push RPC plus plugin HTTP route + +Add an SDK/runtime-host RPC such as `SubmitScanSourceChanges`, update +`silo-server` to implement it, update the ARR plugin to declare `http_routes.v1` +and receive public webhook requests, then have the plugin parse payloads and +push changes to the host. + +**Pros** + +- Provider-specific webhook parsing lives in the provider plugin. +- The host can stay more generic once the push RPC exists. +- Future push-style providers can reuse the same plugin-to-host contract. + +**Cons** + +- Requires a coordinated SDK release, server update, plugin update, and catalog + update before the feature exists. +- Plugin public URLs include installation id, so reinstalling or replacing the + plugin can change callback URLs unless the host adds another stable alias. +- The plugin still needs durable webhook secret/config state or host-provided + secret state. +- More moving parts in the most security-sensitive path. +- The feature cannot ship as a narrow server/API/UI change. + +### Option C: Separate `arr-webhook` plugin with internal buffer + +Create a new plugin that exposes a public webhook route, stores received paths +in an internal buffer, and returns them when the host later polls +`scan_source.v1`. + +**Pros** + +- No SDK push RPC required. +- Provider-specific parsing stays outside the host. +- Reuses the existing pull contract. + +**Cons** + +- Still has poll latency even though arr sent a real-time webhook. +- The plugin needs durable queue/buffer storage, idempotency, and cleanup. The + current plugin model is not designed around plugin-owned durable event queues. +- Reinstalling the plugin risks losing buffered events or changing callback URLs. +- More operationally fragile than letting the host enqueue immediately. +- Creates a second ARR Autoscan plugin with overlapping purpose. + +### Option D: Keep polling only + +Improve docs or setup around current polling, but still require an arr API key. + +**Pros** + +- No implementation work. +- Pulling `/history` remains robust for recovery after Silo downtime. + +**Cons** + +- Does not satisfy the request. +- Still requires API keys. +- Adds latency compared to direct arr webhook delivery. + +## Recommended v1 + +Implement Option A with a built-in source identity. + +Add webhook delivery as a mode on Autoscan source rows. Existing poll sources +remain `delivery_mode = "poll"`. Webhook-only sources use +`delivery_mode = "webhook"` and do not require `connection_id`. + +Webhook sources bind to a **host-discovered built-in source identity**, +`silo.autoscan.arr-webhook`, rather than the installed ARR plugin's capability. +`ScanSourceLister` is a narrow interface with a single adapter +(`PluginScanSourceLister` in `internal/api/autoscan_wiring.go`); wrap it in a +composite lister that appends one synthetic `DiscoveredSource` entry for the +built-in identity. Because the Add-source picker, create-time validation, +labels, and events all consume only `(plugin_id, capability_id, display_name)`, +they work unchanged. The provider is never invoked for webhook sources — +`PollOnce` skips them — so no `PollChanges` implementation exists for the +built-in identity, and none is needed. + +For v1, `delivery_mode = "webhook"` is accepted only for the built-in +`silo.autoscan.arr-webhook` identity, and `delivery_mode = "poll"` is rejected +for it (it has no plugin to poll). Existing ARR plugin sources stay poll-mode. + +## Data model + +Add a Goose migration. + +### `autoscan_sources` + +Add: + +| column | type / default | purpose | +|---|---|---| +| `delivery_mode` | text NOT NULL DEFAULT `'poll'` | `poll` or `webhook`; future `hybrid` can be added later | + +Check constraint: + +```sql +delivery_mode = ANY (ARRAY['poll'::text, 'webhook'::text]) +``` + +Rules: + +- Poll mode keeps today's behavior. +- Webhook mode skips `PollChanges` and does not require a connection. +- Webhook mode is valid only for the built-in `silo.autoscan.arr-webhook` + identity; poll mode is invalid for it. +- Existing rows default to poll. +- `source_config.webhook_provider` may be `sonarr`, `radarr`, or `auto`. + Use `auto` by default and infer from payload shape when possible. + +### `autoscan_webhook_endpoints` + +One active webhook endpoint per source for v1. + +| column | type / default | purpose | +|---|---|---| +| `source_id` | uuid PK REFERENCES `autoscan_sources(id)` ON DELETE CASCADE | webhook owner | +| `secret_hash` | text UNIQUE NOT NULL | lookup hash of the bearer URL token | +| `secret_ref` | text NOT NULL | encrypted token, admin-display only | +| `secret_suffix` | text NOT NULL | last 6-8 chars for UI/debug | +| `created_at` | timestamptz NOT NULL DEFAULT now() | creation time | +| `rotated_at` | timestamptz | last token rotation | +| `last_received_at` | timestamptz | latest valid delivery | +| `last_error_at` | timestamptz | latest parse/ingest error | +| `last_error_message` | text NOT NULL DEFAULT '' | bounded sanitized error | + +Secret handling: + +- Generate at least 32 random bytes and encode URL-safe. +- Store a plain SHA-256 hash for lookup. For a 32-byte random token this is + standard and sufficient (preimage resistance is what matters); an HMAC-keyed + variant adds key-management complexity for no practical gain here. +- Store encrypted `secret_ref` using `internal/secret.Cipher` and AAD bound to + `autoscan_webhook_endpoints.source_id`. +- Never log the token or full URL. + +### `autoscan_webhook_deliveries` + +Persist each actionable parsed delivery before returning `202`. Rows store the +source, provider event type, normalized changes, original receive time, retry +attempt count/backoff, and a bounded lease owner/time. The request attempts the +delivery immediately; transient resolver/database/queue failures release it for +an internal retry task. Workers claim due rows with `FOR UPDATE SKIP LOCKED`, so +multiple API nodes can drain the queue without double-owning a delivery. A +successful consume deletes the row; deleting the source cascades its pending +deliveries. + +## API + +All admin endpoints are additive under `/api/v1/admin/autoscan`. + +### Source response additions + +Add fields to `AutoscanSource` responses: + +```json +{ + "delivery_mode": "poll", + "webhook_configured": true, + "webhook_url": "https://example/api/v1/autoscan/webhooks/...", + "webhook_secret_suffix": "abc123", + "webhook_last_received_at": "2026-07-09T12:34:56Z", + "webhook_last_error_at": null, + "webhook_last_error_message": "" +} +``` + +`webhook_url` is admin-only and stays redisplayable in admin source responses +(the encrypted `secret_ref` exists for exactly this). The token's blast radius +is small — the worst an attacker can do with it is trigger scans of paths that +already resolve inside Silo libraries — so one-time display would buy little +security while costing real operator pain (re-pasting into arr after a config +wipe would force a rotation). + +### Source write additions + +Extend create/update payloads: + +```json +{ + "delivery_mode": "webhook", + "source_config": { + "webhook_provider": "sonarr" + } +} +``` + +Validation: + +- `delivery_mode = "poll"`: current validation stays. If enabled, a connection + should be bound for ARR poll sources. +- `delivery_mode = "webhook"`: only valid with the built-in + `silo.autoscan.arr-webhook` identity. No connection is required. A bound + connection is allowed only for rewrite suggestions. +- The installed-capability check treats the built-in identity as always + available (it comes from the composite lister, not plugin installations). +- `source_config.webhook_provider`: optional; when present must be `sonarr`, + `radarr`, or `auto`. + +### Webhook endpoint management + +Add: + +- `POST /admin/autoscan/sources/{id}/webhook` + - Creates a webhook endpoint if missing and returns URL/status. +- `POST /admin/autoscan/sources/{id}/webhook/rotate` + - Replaces the token, invalidates the old URL, returns the new URL. +- `DELETE /admin/autoscan/sources/{id}/webhook` + - Deletes the endpoint and makes future deliveries return not found. + +### Public delivery endpoint + +Add: + +```http +POST /api/v1/autoscan/webhooks/{token} +Content-Type: application/json +``` + +Responses: + +- `202 Accepted`: valid token; event accepted, ignored, or enqueued. Use 202 for + valid test events and valid disabled sources so arr setup tests do not fail + noisily. +- `400 Bad Request`: valid token but malformed/unsupported payload. +- `404 Not Found`: token not found. Do not reveal whether a source exists. +- `405 Method Not Allowed`: non-POST. +- `413 Payload Too Large`: body exceeds the configured cap. +- `429 Too Many Requests`: per-token or per-IP rate limit exceeded. +- `500 Internal Server Error`: the delivery could not be durably persisted. + +Sonarr/Radarr do not replay failed notification events. Silo must therefore +persist actionable deliveries before `202` and retry ingest failures internally; +HTTP error responses are not a delivery retry mechanism. + +The body should be small; cap at 256 KiB unless real payload fixtures prove that +is too low. + +## Host service design + +Factor the existing `PollOnce` consume path so polling and webhooks share it. + +Proposed shape: + +```go +type ChangeIngest struct { + SourceID string + DeliveryMode string // "poll" or "webhook" + ProviderEventType string + Changes []autoscan.Change + ReceivedAt time.Time +} + +func (s *Service) IngestChanges(ctx context.Context, input ChangeIngest) (IngestResult, error) +``` + +Internally, introduce an unexported helper that both `PollOnce` and +`IngestChanges` call after changes are known: + +```go +func (s *Service) consumeSourceChanges(ctx context.Context, src Source, changes []Change, opts consumeOptions) (consumeResult, error) +``` + +It should reuse: + +- `rewriteChanges` +- `resolveAndClaim` +- `collapseTargetsToLibraryScans` +- `enqueueScanTargets` +- `releaseClaims` +- event creation/finish logic + +Differences from polling: + +- No marker is advanced for webhook deliveries. +- Webhook events use marker fields as empty/null. +- Webhook event creation skips the running-event single-flight exclusion (see + "Security and reliability") — concurrent deliveries each ingest; the + suppressor claim is the duplicate guard. +- `changes_returned`, `changes_resolved`, `targets_claimed`, + `scans_created`, `scans_reused`, and `scans_suppressed` should be recorded + exactly like poll events. +- If any resolve attempt fails transiently, return an error and record the + event as error. Already-enqueued targets are allowed; a duplicate later + delivery is safe. +- If all paths are outside Silo folders, finish as `unresolved`, same as poll. + +`PollOnce` changes: + +- Skip sources with `delivery_mode = "webhook"`. +- Continue current behavior for `delivery_mode = "poll"`. +- If a future `hybrid` mode is added, poll only when a connection is bound. + +## Payload parsing + +Implement parsing in a small ARR-specific host package, for example +`internal/autoscan/arrwebhook`. + +Input: + +```go +func Parse(provider string, body []byte) (ParsedWebhook, error) +``` + +Output: + +```go +type ParsedWebhook struct { + Provider string + EventType string + Test bool + Changes []autoscan.Change +} +``` + +Provider inference: + +- If `source_config.webhook_provider` is `sonarr` or `radarr`, use it. +- If `auto`, infer from top-level `series` vs `movie`, or from top-level field + prefixes if supporting flattened payloads. +- If inference fails on a non-test event, return a 400 with a sanitized message. + +Events: + +- `Test`: no-op success. Update `last_received_at`; do not enqueue. +- `Download`, `Import`, `Upgrade`, `DownloadComplete`: enqueue imported file + paths as `ChangeScopeFile`. +- `Rename`: enqueue new and previous file paths as `ChangeScopeFile`. Previous + paths may no longer exist; `resolveChange` already falls back through + `ResolveVanishedPath` for file changes. +- `EpisodeFileDelete`, `MovieFileDelete`: enqueue deleted file path as + `ChangeScopeFile` when present. +- Unknown event types: valid token, unsupported event. Return 202 no-op and + record a low-noise status rather than making arr mark the webhook unhealthy. + +Path extraction should support the common webhook JSON shapes and should be +covered by fixtures: + +- Sonarr: + - `episodeFile.path` + - `episodeFiles[].path` + - `episodeFile.previousPath`, `episodeFiles[].previousPath` + - `series.path` as a subtree fallback only when no file path exists +- Radarr: + - `movieFile.path` + - `movieFiles[].path` + - `movieFile.previousPath`, `movieFiles[].previousPath` + - `movie.path` or `movie.folderPath` as a subtree fallback only when no file + path exists + +The parser should dedupe exact paths before returning changes. + +## Security and reliability + +- Public endpoint is unauthenticated but token-bearing. +- Token lookup should be constant-shape: invalid tokens return 404 without + distinguishing disabled/deleted sources. +- Valid token but disabled source/global Autoscan disabled should return 202 + and perform no enqueue. This prevents arr retries or setup failures while + respecting Silo's disabled state. +- Apply a per-token rate limit and a coarse per-IP limit. Build on the existing + `internal/ratelimit` middleware (it already supports per-endpoint and per-IP + limits) rather than introducing a new mechanism. +- Use `http.MaxBytesReader` or equivalent for body cap. +- Do not log request bodies. If an error needs context, log only source id, + event type, provider, path count, and sanitized bounded error text. +- Never include token in event logs or regular app logs. +- Preserve current duplicate suppression. Duplicate webhook deliveries should + at worst create suppressed Autoscan events, not duplicate scan storms. +- Honor configured burst capacity in both in-memory and Redis rate-limit + backends; season-pack imports commonly deliver more than one event per second. +- Keep transient ingest failures in the durable delivery queue with bounded + exponential backoff. A short-interval hidden task reclaims due or abandoned + leases and retries them independently of the poll cadence. +- **Webhook deliveries must NOT be single-flighted and dropped.** The + running-event exclusion in `CreateEvent` (advisory lock + running-row check) + exists to stop overlapping polls of the same marker window. Poll cycles can + safely be skipped because the marker window is re-read next cycle; a webhook + delivery has no marker, so a dropped delivery is lost forever. Sonarr fires + one webhook per import in rapid succession (a season pack is many imports in + seconds), so "another event is running" is the normal case during a burst. + `CreateEvent` takes a flag (or a separate path) that skips the running-event + exclusion for webhook deliveries; the duplicate-scan guard for concurrent + ingests is the suppressor's atomic `ShouldScan` claim, which webhook + ingestion reuses. Poll-mode behavior is unchanged. If a `hybrid` mode ships + later, the rule is "poll defers to webhook", never "webhook drops". + +## Admin UI + +Extend `AdminAutoscan` / `SourcesPanel`. + +Source row/edit controls: + +- Delivery mode segmented control: `Poll` / `Webhook`. +- Poll mode: + - Existing connection picker, poll interval, rewrites, sync suggestions. +- Webhook mode: + - Provider selector: `Auto`, `Sonarr`, `Radarr`. + - Webhook URL display with copy button. + - Rotate URL action. + - Last received / last error status. + - Connection picker hidden or clearly optional. If a connection is bound, it + is used only for existing rewrite suggestions, not for ingestion. +- Keep path rewrites visible in both modes; webhook payload paths are arr-side + paths and still need host-side rewrites in Docker/mount-mismatch setups. + +Connections panel: + +- No required changes, except copy should not imply a connection is mandatory + for webhook-only sources. + +Add-source picker: + +- The built-in `silo.autoscan.arr-webhook` entry appears alongside installed + plugin capabilities (supplied by the composite lister) with a display name + like "Sonarr/Radarr Webhook". Picking it creates a webhook-mode source. + +Activity panel: + +- Show delivery mode and provider event type on event rows (the schema adds + both fields in v1). + +## Event metadata (required in v1) + +Add fields to `autoscan_events`: + +| column | type / default | purpose | +|---|---|---| +| `delivery_mode` | text NOT NULL DEFAULT `'poll'` | poll/webhook event source | +| `provider_event_type` | text NOT NULL DEFAULT '' | arr event type | + +These ship in v1, in the same migration as the source/endpoint changes. They +are not required for the enqueue path, but debugging webhook intake without +knowing which delivery produced an event is exactly the pain this feature hits +in week one, and the cost is two columns in a migration being written anyway. + +## Rollout + +1. Add migrations for source delivery mode, webhook endpoint table, and + `autoscan_events` delivery-mode/provider-event-type columns. +2. Add the composite `ScanSourceLister` exposing the built-in + `silo.autoscan.arr-webhook` identity. +3. Add repository methods for webhook endpoint create/get-by-hash/rotate/delete + and source delivery-mode fields, plus the `CreateEvent` no-exclusion path + for webhook deliveries. +4. Refactor `autoscan.Service` to share the consume path and add + `IngestChanges`. +5. Add ARR webhook parser fixtures. +6. Add public route and handler. +7. Add admin endpoint fields and endpoint-management routes. +8. Update frontend types/hooks and `SourcesPanel`. +9. Keep all existing source rows as poll mode. +10. Update docs or setup text after review. + +## Testing + +Backend: + +- Migration applies and rolls back. +- Existing poll-mode service tests still pass. +- `PollOnce` skips webhook-mode sources. +- The composite lister exposes the built-in identity; source creation accepts + it for webhook mode and rejects poll mode for it. +- `IngestChanges` applies rewrites and enqueues the same targets as polling for + equivalent changes. +- Actionable deliveries are persisted before processing; transient failures + return `202`, remain queued, and are consumed by the retry task. +- Delivery leases are ownership-guarded and reclaimable after worker failure. +- Concurrent/burst webhook deliveries for one source all ingest — none are + dropped by the running-event exclusion; duplicate targets are suppressed by + the suppressor claim. +- Duplicate webhook deliveries are suppressed by existing debounce logic. +- Webhook events record `delivery_mode` and `provider_event_type`. +- Disabled source and disabled global setting return accepted/no-op behavior. +- Invalid token returns 404; malformed payload for valid token returns 400. +- Body cap returns 413. +- Rate limit returns 429. +- Redis rate limiting honors the configured webhook burst allowance. +- Upgrade payloads reconcile both the new file and arr's `deletedFiles` paths. +- Token rotation invalidates old token and accepts new token. +- Parser fixture tests for Sonarr Download, Rename, EpisodeFileDelete, Test. +- Parser fixture tests for Radarr Download, Rename, MovieFileDelete, Test. +- Unknown event type is accepted/no-op. +- No logs include the token or raw request body. + +Frontend: + +- Source can switch from poll to webhook mode without selecting a connection. +- Webhook URL renders only for configured endpoint. +- Rotate action replaces the URL. +- Provider selector writes `source_config.webhook_provider`. +- Poll mode behavior remains unchanged. +- Rewrite editor remains available in webhook mode. + +Validation commands: + +```bash +GOWORK=off go test ./internal/autoscan ./internal/api/handlers +cd web && pnpm run lint +cd web && pnpm run format:check +make verify-local-paths +``` + +Broader validation after implementation: + +```bash +GOWORK=off go test ./... +cd web && pnpm run build +``` + +## Resolved review decisions (2026-07-09) + +1. **Built-in source identity.** Webhook mode uses a host-discovered + `silo.autoscan.arr-webhook` identity, not the installed ARR plugin's + capability. Create-time validation requires an installed capability, so + plugin-bound identity would force installing a plugin the webhook path never + calls; the composite-lister alternative is ~20 lines and everything + downstream consumes only `(plugin_id, capability_id, display_name)`. +2. **Webhook URL stays redisplayable** in admin-only responses. Low token blast + radius (worst case: spurious scans of paths already inside Silo libraries); + one-time display would force rotations after any config loss. +3. **No `hybrid` in v1.** Purely additive later; shipping it now forces + poll/webhook overlap semantics to be solved before any field experience. +4. **Unknown event types return 202 no-op.** Arr adds event types over time; a + 400 makes arr mark the webhook unhealthy and nag the operator about + something Silo intentionally ignores. +5. **`autoscan_events.delivery_mode` and `provider_event_type` ship in v1.** + Two columns in a migration being written anyway; needed for week-one + debugging of webhook intake. +6. **Parser lives in `internal/autoscan/arrwebhook`.** Fixture-testable, keeps + handlers thin, matches the repo's domain-package organization. +7. **Webhook deliveries are never single-flight-dropped** (raised during + review, not an original question). Poll skips are safe because the marker + window is re-read; webhooks have no marker, and Sonarr bursts (season packs) + make a running event the normal case. See "Security and reliability". + +## References + +- Current Autoscan plugin architecture: + `docs/superpowers/specs/2026-06-02-autoscan-plugin-architecture-design.md` +- Current ARR polling design: + `docs/superpowers/specs/2026-06-02-autoscan-arr-polling-design.md` +- Current rewrite sync design: + `docs/superpowers/specs/2026-06-02-autoscan-rewrite-sync-design.md` +- Current SDK pull contract: sibling `silo-plugin-sdk` repo, + `proto/silo/plugin/v1/scan_source.proto` +- Current ARR plugin: sibling `silo-plugin-autoscan-arr` repo +- Servarr path/event references: + - Sonarr custom scripts: + - Radarr custom scripts: diff --git a/internal/activitylog/middleware.go b/internal/activitylog/middleware.go index ff6b9fda..fa9402c3 100644 --- a/internal/activitylog/middleware.go +++ b/internal/activitylog/middleware.go @@ -119,6 +119,7 @@ func NewMiddleware(w Writer, nodeID string) func(http.Handler) http.Handler { pathPattern = route } } + path = RedactSecretPathParams(r, path) // After handler chain completes, auth middleware has populated lc entry := LogEntry{ @@ -143,6 +144,33 @@ func NewMiddleware(w Writer, nodeID string) func(http.Handler) http.Handler { } } +// RedactSecretPathParams strips bearer credentials from a request path before +// it reaches any log sink. Public webhook endpoints authenticate via a secret +// path segment (chi route params named "token" or "secret" — autoscan webhook +// intake, webhook-sync); logging the raw path would persist the credential in +// app logs and the activity table. Must be called after the handler chain ran +// so the chi route context is populated. +func RedactSecretPathParams(r *http.Request, path string) string { + routeCtx := chi.RouteContext(r.Context()) + if routeCtx == nil { + return path + } + for i, key := range routeCtx.URLParams.Keys { + if key != "token" && key != "secret" { + continue + } + if i >= len(routeCtx.URLParams.Values) { + continue + } + value := routeCtx.URLParams.Values[i] + if value == "" { + continue + } + path = strings.Replace(path, "/"+value, "/[redacted]", 1) + } + return path +} + // isStreamChunk returns true if the path looks like a stream segment/manifest // chunk rather than a session-start request. func isStreamChunk(path string) bool { diff --git a/internal/activitylog/redact_test.go b/internal/activitylog/redact_test.go new file mode 100644 index 00000000..9793d671 --- /dev/null +++ b/internal/activitylog/redact_test.go @@ -0,0 +1,58 @@ +package activitylog + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi/v5" +) + +func TestRedactSecretPathParams(t *testing.T) { + cases := []struct { + name string + params map[string]string + path string + want string + }{ + { + name: "autoscan webhook token", + params: map[string]string{"token": "2riBxLS4UcC8w8WdRoge8F0f8LOLkz4yPsPNA01gWBY"}, + path: "/api/v1/autoscan/webhooks/2riBxLS4UcC8w8WdRoge8F0f8LOLkz4yPsPNA01gWBY", + want: "/api/v1/autoscan/webhooks/[redacted]", + }, + { + name: "webhook-sync secret", + params: map[string]string{"secret": "c98bffdbe95b5be34141e2a7f0327e1f"}, + path: "/api/v1/webhook-sync/webhooks/c98bffdbe95b5be34141e2a7f0327e1f", + want: "/api/v1/webhook-sync/webhooks/[redacted]", + }, + { + name: "non-secret params untouched", + params: map[string]string{"id": "abc123"}, + path: "/api/v1/admin/autoscan/sources/abc123", + want: "/api/v1/admin/autoscan/sources/abc123", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest("POST", tc.path, nil) + routeCtx := chi.NewRouteContext() + for k, v := range tc.params { + routeCtx.URLParams.Add(k, v) + } + r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, routeCtx)) + + if got := RedactSecretPathParams(r, tc.path); got != tc.want { + t.Fatalf("redacted path = %q, want %q", got, tc.want) + } + }) + } +} + +func TestRedactSecretPathParamsNoRouteContext(t *testing.T) { + r := httptest.NewRequest("POST", "/api/v1/autoscan/webhooks/tok", nil) + if got := RedactSecretPathParams(r, r.URL.Path); got != r.URL.Path { + t.Fatalf("path without route context must pass through, got %q", got) + } +} diff --git a/internal/api/autoscan_wiring.go b/internal/api/autoscan_wiring.go index 86c812a9..a78e8e72 100644 --- a/internal/api/autoscan_wiring.go +++ b/internal/api/autoscan_wiring.go @@ -149,7 +149,10 @@ func BuildAutoscanService( scantrigger.NewResolver(folderRepo), queue, autoscan.NewRedisSuppressor(redisClient), - PluginScanSourceLister{installationStore}, + autoscan.WithBuiltinSources( + PluginScanSourceLister{installationStore}, + autoscan.BuiltinArrWebhookSource(), + ), ) // Wire the connection-test + rewrite-suggester deps: a (long-timeout) // arr root-folder/status client and a Silo media-folder lister. diff --git a/internal/api/handlers/autoscan.go b/internal/api/handlers/autoscan.go index ca005667..c0852f6e 100644 --- a/internal/api/handlers/autoscan.go +++ b/internal/api/handlers/autoscan.go @@ -39,6 +39,15 @@ type autoscanStore interface { ListRunningEvents(ctx context.Context) ([]autoscan.Event, error) GetQueueSummary(ctx context.Context) (autoscan.QueueSummary, error) LatestEventAt(ctx context.Context) (*time.Time, error) + CreateWebhookEndpoint(ctx context.Context, sourceID string) (autoscan.WebhookEndpoint, string, error) + RotateWebhookEndpoint(ctx context.Context, sourceID string) (autoscan.WebhookEndpoint, string, error) + DeleteWebhookEndpoint(ctx context.Context, sourceID string) error + GetWebhookEndpoint(ctx context.Context, sourceID string) (autoscan.WebhookEndpoint, error) + ListWebhookEndpoints(ctx context.Context) ([]autoscan.WebhookEndpoint, error) + RevealWebhookToken(ctx context.Context, sourceID string) (string, error) + ResolveWebhookToken(ctx context.Context, token string) (autoscan.Source, autoscan.WebhookEndpoint, error) + TouchWebhookReceived(ctx context.Context, sourceID string) error + RecordWebhookError(ctx context.Context, sourceID, msg string) error } // autoscanTriggerer is the subset of *autoscan.Service the handler needs. @@ -55,6 +64,9 @@ type autoscanTriggerer interface { // SuggestRewrites proposes path rewrites by matching the source's arr root // folders against Silo's media folders. SuggestRewrites(ctx context.Context, sourceID string) (autoscan.RewriteSuggestions, error) + // IngestChanges feeds webhook-delivered changes through the shared + // rewrite/resolve/suppress/enqueue pipeline. + IngestChanges(ctx context.Context, in autoscan.ChangeIngest) (autoscan.IngestResult, error) } // autoscanTriggerUpdater reconfigures the poll task's schedule when the default @@ -69,11 +81,14 @@ type autoscanTriggerUpdater interface { const autoscanPollTaskKey = "autoscan_poll" // AutoscanHandler serves the admin-only autoscan settings/connections/sources/ -// trigger API. +// trigger API plus the public webhook delivery endpoint. type AutoscanHandler struct { repo autoscanStore svc autoscanTriggerer triggers autoscanTriggerUpdater // optional; nil skips rescheduling + // publicURL is the server's externally reachable base URL; when empty, + // webhook URLs are returned as absolute paths for the UI to complete. + publicURL string } func NewAutoscanHandler(repo autoscanStore, svc autoscanTriggerer) *AutoscanHandler { @@ -88,6 +103,13 @@ func (h *AutoscanHandler) SetTriggerUpdater(t autoscanTriggerUpdater) { h.triggers = t } +// SetPublicURL wires the server's public base URL so webhook URLs are returned +// fully qualified. Optional: when unset, responses carry the absolute path and +// the admin UI prefixes its own origin. +func (h *AutoscanHandler) SetPublicURL(publicURL string) { + h.publicURL = strings.TrimRight(strings.TrimSpace(publicURL), "/") +} + // --- Settings --- type autoscanSettingsResponse struct { @@ -307,19 +329,29 @@ func (h *AutoscanHandler) HandleDeleteConnection(w http.ResponseWriter, r *http. // --- Sources --- // autoscanSourceResponse is the client view of a source. It carries no -// credentials: the connection link is by id only. +// credentials: the connection link is by id only. The webhook_* fields are +// admin-only setup surface; webhook_url deliberately stays redisplayable (the +// token's blast radius is spurious scans of already-registered library paths, +// and one-time display would force a rotation after any config loss). type autoscanSourceResponse struct { - ID string `json:"id"` - PluginID string `json:"plugin_id"` - CapabilityID string `json:"capability_id"` - ConnectionID *string `json:"connection_id"` - Enabled bool `json:"enabled"` - PollIntervalSeconds *int `json:"poll_interval_seconds,omitempty"` - PathRewrites []autoscan.PathRewrite `json:"path_rewrites"` - SourceConfig map[string]string `json:"source_config"` - Label string `json:"label"` - LastRunAt *time.Time `json:"last_run_at,omitempty"` - LastError *string `json:"last_error,omitempty"` + ID string `json:"id"` + PluginID string `json:"plugin_id"` + CapabilityID string `json:"capability_id"` + ConnectionID *string `json:"connection_id"` + Enabled bool `json:"enabled"` + DeliveryMode string `json:"delivery_mode"` + PollIntervalSeconds *int `json:"poll_interval_seconds,omitempty"` + PathRewrites []autoscan.PathRewrite `json:"path_rewrites"` + SourceConfig map[string]string `json:"source_config"` + Label string `json:"label"` + LastRunAt *time.Time `json:"last_run_at,omitempty"` + LastError *string `json:"last_error,omitempty"` + WebhookConfigured bool `json:"webhook_configured"` + WebhookURL string `json:"webhook_url,omitempty"` + WebhookSecretSuffix string `json:"webhook_secret_suffix,omitempty"` + WebhookLastReceivedAt *time.Time `json:"webhook_last_received_at,omitempty"` + WebhookLastErrorAt *time.Time `json:"webhook_last_error_at,omitempty"` + WebhookLastErrorMessage string `json:"webhook_last_error_message,omitempty"` } func sourceResponse(s autoscan.Source) autoscanSourceResponse { @@ -330,12 +362,17 @@ func sourceResponse(s autoscan.Source) autoscanSourceResponse { rewrites = []autoscan.PathRewrite{} } config := normalizeSourceConfig(s.SourceConfig) + deliveryMode := s.DeliveryMode + if deliveryMode == "" { + deliveryMode = autoscan.DeliveryModePoll + } return autoscanSourceResponse{ ID: s.ID, PluginID: s.PluginID, CapabilityID: s.CapabilityID, ConnectionID: s.ConnectionID, Enabled: s.Enabled, + DeliveryMode: deliveryMode, PollIntervalSeconds: s.PollIntervalSeconds, PathRewrites: rewrites, SourceConfig: config, @@ -345,6 +382,45 @@ func sourceResponse(s autoscan.Source) autoscanSourceResponse { } } +// webhookURLFor builds the delivery URL for a token: fully qualified when the +// server knows its public URL, otherwise an absolute path the admin UI +// completes with its own origin. +func (h *AutoscanHandler) webhookURLFor(token string) string { + return h.publicURL + "/api/v1/autoscan/webhooks/" + token +} + +// attachWebhookState decorates a source response with its endpoint status and +// redisplayable URL. A reveal failure (e.g. cipher/key drift) degrades to +// status-without-URL rather than failing the response. +func (h *AutoscanHandler) attachWebhookState(ctx context.Context, resp *autoscanSourceResponse, endpoint autoscan.WebhookEndpoint) { + resp.WebhookConfigured = true + resp.WebhookSecretSuffix = endpoint.SecretSuffix + resp.WebhookLastReceivedAt = endpoint.LastReceivedAt + resp.WebhookLastErrorAt = endpoint.LastErrorAt + resp.WebhookLastErrorMessage = endpoint.LastErrorMessage + token, err := h.repo.RevealWebhookToken(ctx, endpoint.SourceID) + if err != nil { + slog.WarnContext(ctx, "autoscan: reveal webhook token failed", "component", "api", "source_id", endpoint.SourceID, "err", err) + return + } + resp.WebhookURL = h.webhookURLFor(token) +} + +// sourceResponseWithWebhook builds a single source response including webhook +// endpoint state (used by the single-source create/update/webhook handlers). +func (h *AutoscanHandler) sourceResponseWithWebhook(ctx context.Context, s autoscan.Source) autoscanSourceResponse { + resp := sourceResponse(s) + endpoint, err := h.repo.GetWebhookEndpoint(ctx, s.ID) + if err != nil { + if !errors.Is(err, autoscan.ErrNotFound) { + slog.WarnContext(ctx, "autoscan: load webhook endpoint failed", "component", "api", "source_id", s.ID, "err", err) + } + return resp + } + h.attachWebhookState(ctx, &resp, endpoint) + return resp +} + func (h *AutoscanHandler) HandleListSources(w http.ResponseWriter, r *http.Request) { // Sources are operator-created (no auto-seed): just list what exists. Use // GET /scan-source-plugins for the Add-source picker of installed capabilities. @@ -353,9 +429,23 @@ func (h *AutoscanHandler) HandleListSources(w http.ResponseWriter, r *http.Reque writeAutoscanError(w, err) return } + // One batched endpoint query for the whole listing (not per source). + endpoints, err := h.repo.ListWebhookEndpoints(r.Context()) + if err != nil { + writeAutoscanError(w, err) + return + } + endpointBySource := make(map[string]autoscan.WebhookEndpoint, len(endpoints)) + for _, e := range endpoints { + endpointBySource[e.SourceID] = e + } out := make([]autoscanSourceResponse, 0, len(sources)) for _, s := range sources { - out = append(out, sourceResponse(s)) + resp := sourceResponse(s) + if endpoint, ok := endpointBySource[s.ID]; ok { + h.attachWebhookState(r.Context(), &resp, endpoint) + } + out = append(out, resp) } writeJSON(w, http.StatusOK, struct { Sources []autoscanSourceResponse `json:"sources"` @@ -402,12 +492,55 @@ type autoscanCreateSourceInput struct { CapabilityID string `json:"capability_id"` ConnectionID *string `json:"connection_id"` Enabled bool `json:"enabled"` + DeliveryMode string `json:"delivery_mode"` PollIntervalSeconds *int `json:"poll_interval_seconds"` PathRewrites []autoscan.PathRewrite `json:"path_rewrites"` SourceConfig map[string]string `json:"source_config"` Label string `json:"label"` } +// resolveDeliveryMode validates a requested delivery mode against the source +// identity: webhook mode exists only for the built-in ARR webhook identity +// (which cannot poll — it has no plugin), and an empty mode defaults to +// whichever mode the identity supports. +func resolveDeliveryMode(requested, pluginID, capabilityID string) (string, error) { + isBuiltinWebhook := autoscan.IsBuiltinArrWebhookIdentity(pluginID, capabilityID) + switch strings.ToLower(strings.TrimSpace(requested)) { + case "": + if isBuiltinWebhook { + return autoscan.DeliveryModeWebhook, nil + } + return autoscan.DeliveryModePoll, nil + case autoscan.DeliveryModePoll: + if isBuiltinWebhook { + return "", fmt.Errorf("the built-in Sonarr/Radarr webhook source has no plugin to poll; delivery_mode must be webhook") + } + return autoscan.DeliveryModePoll, nil + case autoscan.DeliveryModeWebhook: + if !isBuiltinWebhook { + return "", fmt.Errorf("delivery_mode webhook is only valid for the built-in Sonarr/Radarr webhook source (%s/%s)", + autoscan.BuiltinArrWebhookPluginID, autoscan.BuiltinArrWebhookCapabilityID) + } + return autoscan.DeliveryModeWebhook, nil + default: + return "", fmt.Errorf("delivery_mode must be poll or webhook") + } +} + +// validateWebhookProvider checks the optional source_config.webhook_provider +// value used by webhook payload parsing. +func validateWebhookProvider(config map[string]string) error { + provider, ok := config["webhook_provider"] + if !ok { + return nil + } + switch strings.ToLower(strings.TrimSpace(provider)) { + case "", "auto", "sonarr", "radarr": + return nil + } + return fmt.Errorf("source_config.webhook_provider must be auto, sonarr, or radarr") +} + // HandleCreateSource creates a new source bound to an installed scan_source // capability. Many sources may share one capability (e.g. one Sonarr plugin // fronting several arr servers, one source per connection). The capability must @@ -436,9 +569,25 @@ func (h *AutoscanHandler) HandleCreateSource(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusBadRequest, "bad_request", err.Error()) return } + deliveryMode, err := resolveDeliveryMode(in.DeliveryMode, pluginID, capID) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + sourceConfig := normalizeSourceConfig(in.SourceConfig) + if deliveryMode == autoscan.DeliveryModeWebhook { + if err := validateWebhookProvider(sourceConfig); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + if provider, ok := sourceConfig["webhook_provider"]; ok { + sourceConfig["webhook_provider"] = strings.ToLower(strings.TrimSpace(provider)) + } + } // Validate the (plugin_id, capability_id) is a currently-installed // scan_source capability — a source may only be created against an installed - // plugin capability. + // plugin capability. The built-in webhook identity passes because the + // composite lister always supplies it. available, err := h.svc.ListAvailableScanSources(r.Context()) if err != nil { writeAutoscanError(w, err) @@ -454,9 +603,10 @@ func (h *AutoscanHandler) HandleCreateSource(w http.ResponseWriter, r *http.Requ CapabilityID: capID, ConnectionID: connArg, Enabled: in.Enabled, + DeliveryMode: deliveryMode, PollIntervalSeconds: in.PollIntervalSeconds, PathRewrites: normalizePathRewrites(in.PathRewrites), - SourceConfig: normalizeSourceConfig(in.SourceConfig), + SourceConfig: sourceConfig, Label: autoscan.NormalizeSourceLabel(in.Label), }) if err != nil { @@ -491,6 +641,7 @@ func scanSourceInstalled(available []autoscan.AvailableScanSource, pluginID, cap type autoscanSourceInput struct { ConnectionID *string `json:"connection_id"` Enabled bool `json:"enabled"` + DeliveryMode string `json:"delivery_mode"` PollIntervalSeconds *int `json:"poll_interval_seconds"` PathRewrites []autoscan.PathRewrite `json:"path_rewrites"` SourceConfig map[string]string `json:"source_config"` @@ -549,6 +700,32 @@ func (h *AutoscanHandler) HandleUpdateSource(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusBadRequest, "bad_request", err.Error()) return } + // Delivery mode is validated against the existing row's immutable identity: + // an empty input keeps the stored mode. + existing, err := h.repo.GetSource(r.Context(), id) + if err != nil { + writeAutoscanError(w, err) + return + } + requestedMode := in.DeliveryMode + if strings.TrimSpace(requestedMode) == "" { + requestedMode = existing.DeliveryMode + } + deliveryMode, err := resolveDeliveryMode(requestedMode, existing.PluginID, existing.CapabilityID) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + sourceConfig := normalizeSourceConfig(in.SourceConfig) + if deliveryMode == autoscan.DeliveryModeWebhook { + if err := validateWebhookProvider(sourceConfig); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + if provider, ok := sourceConfig["webhook_provider"]; ok { + sourceConfig["webhook_provider"] = strings.ToLower(strings.TrimSpace(provider)) + } + } // connection_id is a full-state field: nil means unbind, a UUID string // means bind. A whitespace-only string is normalised to nil (unbound). connArg := normalizeConnectionID(in.ConnectionID) @@ -558,16 +735,17 @@ func (h *AutoscanHandler) HandleUpdateSource(w http.ResponseWriter, r *http.Requ ID: id, ConnectionID: connArg, Enabled: in.Enabled, + DeliveryMode: deliveryMode, PollIntervalSeconds: in.PollIntervalSeconds, PathRewrites: normalizePathRewrites(in.PathRewrites), - SourceConfig: normalizeSourceConfig(in.SourceConfig), + SourceConfig: sourceConfig, Label: autoscan.NormalizeSourceLabel(in.Label), }) if err != nil { writeAutoscanError(w, err) return } - writeJSON(w, http.StatusOK, sourceResponse(updated)) + writeJSON(w, http.StatusOK, h.sourceResponseWithWebhook(r.Context(), updated)) } // normalizeConnectionID maps a full-state connection_id input to a stored value: @@ -719,6 +897,8 @@ type autoscanEventResponse struct { CompletedAt time.Time `json:"completed_at"` DurationMS int64 `json:"duration_ms"` Status string `json:"status"` + DeliveryMode string `json:"delivery_mode"` + ProviderEvent string `json:"provider_event_type,omitempty"` ChangesReturned int `json:"changes_returned"` ChangesResolved int `json:"changes_resolved"` TargetsClaimed int `json:"targets_claimed"` @@ -762,6 +942,8 @@ func eventResponse(event autoscan.EventWithRuns) autoscanEventResponse { CompletedAt: e.CompletedAt, DurationMS: e.DurationMS, Status: string(e.Status), + DeliveryMode: e.DeliveryMode, + ProviderEvent: e.ProviderEventType, ChangesReturned: e.ChangesReturned, ChangesResolved: e.ChangesResolved, TargetsClaimed: e.TargetsClaimed, diff --git a/internal/api/handlers/autoscan_test.go b/internal/api/handlers/autoscan_test.go index 761aeb19..58798de3 100644 --- a/internal/api/handlers/autoscan_test.go +++ b/internal/api/handlers/autoscan_test.go @@ -34,6 +34,16 @@ type fakeAutoscanStore struct { listRunningFn func() ([]autoscan.Event, error) queueSummaryFn func() (autoscan.QueueSummary, error) latestEventAtFn func() (*time.Time, error) + + createWebhookFn func(string) (autoscan.WebhookEndpoint, string, error) + rotateWebhookFn func(string) (autoscan.WebhookEndpoint, string, error) + deleteWebhookFn func(string) error + getWebhookFn func(string) (autoscan.WebhookEndpoint, error) + listWebhooksFn func() ([]autoscan.WebhookEndpoint, error) + revealTokenFn func(string) (string, error) + resolveTokenFn func(string) (autoscan.Source, autoscan.WebhookEndpoint, error) + touchedSources []string + webhookErrs map[string]string } func (f *fakeAutoscanStore) GetSettings(context.Context) (autoscan.Settings, error) { @@ -162,6 +172,68 @@ func (f *fakeAutoscanStore) LatestEventAt(context.Context) (*time.Time, error) { return nil, nil } +func (f *fakeAutoscanStore) CreateWebhookEndpoint(_ context.Context, sourceID string) (autoscan.WebhookEndpoint, string, error) { + if f.createWebhookFn != nil { + return f.createWebhookFn(sourceID) + } + return autoscan.WebhookEndpoint{SourceID: sourceID}, "", nil +} + +func (f *fakeAutoscanStore) RotateWebhookEndpoint(_ context.Context, sourceID string) (autoscan.WebhookEndpoint, string, error) { + if f.rotateWebhookFn != nil { + return f.rotateWebhookFn(sourceID) + } + return autoscan.WebhookEndpoint{SourceID: sourceID}, "", nil +} + +func (f *fakeAutoscanStore) DeleteWebhookEndpoint(_ context.Context, sourceID string) error { + if f.deleteWebhookFn != nil { + return f.deleteWebhookFn(sourceID) + } + return nil +} + +func (f *fakeAutoscanStore) GetWebhookEndpoint(_ context.Context, sourceID string) (autoscan.WebhookEndpoint, error) { + if f.getWebhookFn != nil { + return f.getWebhookFn(sourceID) + } + return autoscan.WebhookEndpoint{}, autoscan.ErrNotFound +} + +func (f *fakeAutoscanStore) ListWebhookEndpoints(context.Context) ([]autoscan.WebhookEndpoint, error) { + if f.listWebhooksFn != nil { + return f.listWebhooksFn() + } + return nil, nil +} + +func (f *fakeAutoscanStore) RevealWebhookToken(_ context.Context, sourceID string) (string, error) { + if f.revealTokenFn != nil { + return f.revealTokenFn(sourceID) + } + return "", autoscan.ErrNotFound +} + +func (f *fakeAutoscanStore) ResolveWebhookToken(_ context.Context, token string) (autoscan.Source, autoscan.WebhookEndpoint, error) { + if f.resolveTokenFn != nil { + return f.resolveTokenFn(token) + } + return autoscan.Source{}, autoscan.WebhookEndpoint{}, autoscan.ErrNotFound +} + +func (f *fakeAutoscanStore) TouchWebhookReceived(_ context.Context, sourceID string) error { + f.touchedSources = append(f.touchedSources, sourceID) + return nil +} + +func (f *fakeAutoscanStore) RecordWebhookError(_ context.Context, sourceID, msg string) error { + if f.webhookErrs == nil { + f.webhookErrs = map[string]string{} + } + f.webhookErrs[sourceID] = msg + return nil +} + // fakeTriggerUpdater records the last UpdateTriggers call so a test can assert // the handler reschedules the poll task with the new interval. type fakeTriggerUpdater struct { @@ -197,6 +269,11 @@ type fakeAutoscanTriggerer struct { // send happens-before the test's receive, so reading `called` afterwards is // race-free. done chan struct{} + // ingested records IngestChanges calls; ingestResult/ingestErr drive the + // response. + ingested []autoscan.ChangeIngest + ingestResult autoscan.IngestResult + ingestErr error } func (f *fakeAutoscanTriggerer) PollOnce(context.Context) error { @@ -223,6 +300,11 @@ func (f *fakeAutoscanTriggerer) SuggestRewrites(context.Context, string) (autosc return f.suggestions, f.suggestErr } +func (f *fakeAutoscanTriggerer) IngestChanges(_ context.Context, in autoscan.ChangeIngest) (autoscan.IngestResult, error) { + f.ingested = append(f.ingested, in) + return f.ingestResult, f.ingestErr +} + func newAutoscanRequest(method, target, body, id string) *http.Request { var r *http.Request if body != "" { diff --git a/internal/api/handlers/autoscan_webhook.go b/internal/api/handlers/autoscan_webhook.go new file mode 100644 index 00000000..353252e3 --- /dev/null +++ b/internal/api/handlers/autoscan_webhook.go @@ -0,0 +1,182 @@ +package handlers + +import ( + "errors" + "io" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/autoscan" + "github.com/Silo-Server/silo-server/internal/autoscan/arrwebhook" +) + +// maxWebhookBodyBytes caps public webhook request bodies. Real Sonarr/Radarr +// payloads are a few KiB; season packs with many episode files stay well under +// this. +const maxWebhookBodyBytes = 256 * 1024 + +// --- Admin webhook endpoint management --- + +// HandleCreateSourceWebhook creates the source's webhook endpoint if missing +// and returns the source view including the delivery URL. +// POST /admin/autoscan/sources/{id}/webhook +func (h *AutoscanHandler) HandleCreateSourceWebhook(w http.ResponseWriter, r *http.Request) { + id := strings.TrimSpace(chi.URLParam(r, "id")) + source, err := h.repo.GetSource(r.Context(), id) + if err != nil { + writeAutoscanError(w, err) + return + } + if source.DeliveryMode != autoscan.DeliveryModeWebhook { + writeError(w, http.StatusBadRequest, "bad_request", "source is not in webhook delivery mode") + return + } + if _, _, err := h.repo.CreateWebhookEndpoint(r.Context(), id); err != nil { + writeAutoscanError(w, err) + return + } + writeJSON(w, http.StatusOK, h.sourceResponseWithWebhook(r.Context(), source)) +} + +// HandleRotateSourceWebhook replaces the endpoint's token; the old URL stops +// working immediately. +// POST /admin/autoscan/sources/{id}/webhook/rotate +func (h *AutoscanHandler) HandleRotateSourceWebhook(w http.ResponseWriter, r *http.Request) { + id := strings.TrimSpace(chi.URLParam(r, "id")) + source, err := h.repo.GetSource(r.Context(), id) + if err != nil { + writeAutoscanError(w, err) + return + } + if _, _, err := h.repo.RotateWebhookEndpoint(r.Context(), id); err != nil { + writeAutoscanError(w, err) + return + } + writeJSON(w, http.StatusOK, h.sourceResponseWithWebhook(r.Context(), source)) +} + +// HandleDeleteSourceWebhook removes the endpoint; future deliveries to its URL +// return 404. +// DELETE /admin/autoscan/sources/{id}/webhook +func (h *AutoscanHandler) HandleDeleteSourceWebhook(w http.ResponseWriter, r *http.Request) { + id := strings.TrimSpace(chi.URLParam(r, "id")) + if err := h.repo.DeleteWebhookEndpoint(r.Context(), id); err != nil { + writeAutoscanError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// --- Public delivery endpoint --- + +// webhookAccepted is the 202 body for every accepted delivery (including +// no-ops), so senders can't distinguish disabled sources from active ones. +type webhookAccepted struct { + Status string `json:"status"` +} + +// HandleWebhookDelivery accepts a Sonarr/Radarr webhook POST. The bearer token +// in the path authenticates the delivery. Responses are deliberately coarse: +// 404 for any unknown token (no existence hints), 202 for anything accepted — +// including test events, unsupported event types, and disabled sources — so +// arr never marks a configured webhook unhealthy for a Silo-side state it +// cannot act on. Actionable deliveries are durably queued before the 202, so +// transient ingest failures are retried inside Silo rather than delegated to +// arr (which does not replay failed notification events). +// +// The token, request URL, and body are never logged. +func (h *AutoscanHandler) HandleWebhookDelivery(w http.ResponseWriter, r *http.Request) { + receivedAt := time.Now() + token := chi.URLParam(r, "token") + + source, _, err := h.repo.ResolveWebhookToken(r.Context(), token) + if err != nil { + if errors.Is(err, autoscan.ErrNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Not found") + return + } + writeAutoscanError(w, err) + return + } + + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxWebhookBodyBytes)) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + writeError(w, http.StatusRequestEntityTooLarge, "payload_too_large", "Request body exceeds the webhook payload cap") + return + } + writeError(w, http.StatusBadRequest, "bad_request", "Could not read request body") + return + } + + parsed, err := arrwebhook.Parse(source.SourceConfig["webhook_provider"], body) + if err != nil { + // Parse errors carry no payload content; recording and returning them + // is safe. The token stays out of every message. + _ = h.repo.RecordWebhookError(r.Context(), source.ID, err.Error()) + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + + // Test events, unsupported event types, disabled sources, and globally + // disabled autoscan all acknowledge without enqueueing. Stamping + // last_received_at on every valid delivery keeps the admin "webhook is + // wired up" signal honest even while disabled. + settings, err := h.repo.GetSettings(r.Context()) + if err != nil { + writeAutoscanError(w, err) + return + } + if terr := h.repo.TouchWebhookReceived(r.Context(), source.ID); terr != nil { + slog.WarnContext(r.Context(), "autoscan: touch webhook received failed", "component", "api", "source_id", source.ID, "err", terr) + } + if parsed.Test || len(parsed.Changes) == 0 || !settings.Enabled || !source.Enabled { + writeJSON(w, http.StatusAccepted, webhookAccepted{Status: "accepted"}) + return + } + + result, err := h.svc.IngestChanges(r.Context(), autoscan.ChangeIngest{ + SourceID: source.ID, + ProviderEventType: parsed.EventType, + Changes: parsed.Changes, + ReceivedAt: receivedAt, + }) + if err != nil { + // The only error from IngestChanges is failure to durably accept the + // delivery. Processing errors are queued internally and return Pending. + slog.WarnContext(r.Context(), "autoscan: webhook delivery acceptance failed", "component", "api", + "source_id", source.ID, + "provider", parsed.Provider, + "event_type", parsed.EventType, + "paths", len(parsed.Changes), + "err", err, + ) + _ = h.repo.RecordWebhookError(r.Context(), source.ID, err.Error()) + writeError(w, http.StatusInternalServerError, "internal_error", "Could not durably accept delivery") + return + } + if result.Pending { + slog.WarnContext(r.Context(), "autoscan: webhook delivery queued for retry", "component", "api", + "source_id", source.ID, + "provider", parsed.Provider, + "event_type", parsed.EventType, + "paths", len(parsed.Changes), + ) + } + slog.DebugContext(r.Context(), "autoscan: webhook delivery ingested", "component", "api", + "source_id", source.ID, + "provider", parsed.Provider, + "event_type", parsed.EventType, + "paths", len(parsed.Changes), + "enqueued", result.Enqueued, + "suppressed", result.Suppressed, + "unresolved", result.Unresolved, + "pending", result.Pending, + ) + writeJSON(w, http.StatusAccepted, webhookAccepted{Status: "accepted"}) +} diff --git a/internal/api/handlers/autoscan_webhook_test.go b/internal/api/handlers/autoscan_webhook_test.go new file mode 100644 index 00000000..2746a9a5 --- /dev/null +++ b/internal/api/handlers/autoscan_webhook_test.go @@ -0,0 +1,424 @@ +package handlers + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/autoscan" +) + +const testWebhookToken = "wh_test_token_abcdef123456" + +func webhookSource(enabled bool) autoscan.Source { + return autoscan.Source{ + ID: "src-1", + PluginID: autoscan.BuiltinArrWebhookPluginID, + CapabilityID: autoscan.BuiltinArrWebhookCapabilityID, + Enabled: enabled, + DeliveryMode: autoscan.DeliveryModeWebhook, + SourceConfig: map[string]string{"webhook_provider": "auto"}, + } +} + +// webhookStore returns a store whose resolveTokenFn accepts testWebhookToken. +func webhookStore(source autoscan.Source, settingsEnabled bool) *fakeAutoscanStore { + return &fakeAutoscanStore{ + getSettingsFn: func() (autoscan.Settings, error) { + return autoscan.Settings{Enabled: settingsEnabled, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60}, nil + }, + resolveTokenFn: func(token string) (autoscan.Source, autoscan.WebhookEndpoint, error) { + if token == testWebhookToken { + return source, autoscan.WebhookEndpoint{SourceID: source.ID}, nil + } + return autoscan.Source{}, autoscan.WebhookEndpoint{}, autoscan.ErrNotFound + }, + } +} + +func newWebhookDeliveryRequest(token, body string) *http.Request { + r := httptest.NewRequest(http.MethodPost, "/api/v1/autoscan/webhooks/"+token, strings.NewReader(body)) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("token", token) + return r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, routeCtx)) +} + +const sonarrDownloadBody = `{ + "eventType": "Download", + "series": {"path": "/data/tv/Show"}, + "episodeFile": {"path": "/data/tv/Show/Season 01/e01.mkv"} +}` + +func TestWebhookDeliveryIngestsAndAccepts(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{ingestResult: autoscan.IngestResult{Enqueued: 1}} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, sonarrDownloadBody)) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202 (%s)", rec.Code, rec.Body.String()) + } + if len(svc.ingested) != 1 { + t.Fatalf("ingested = %+v, want one call", svc.ingested) + } + in := svc.ingested[0] + if in.SourceID != "src-1" || in.ProviderEventType != "Download" || len(in.Changes) != 1 { + t.Fatalf("unexpected ingest input: %+v", in) + } + if in.Changes[0].SourcePath != "/data/tv/Show/Season 01/e01.mkv" { + t.Fatalf("change path = %q", in.Changes[0].SourcePath) + } + if len(store.touchedSources) != 1 || store.touchedSources[0] != "src-1" { + t.Fatalf("touched = %+v, want src-1", store.touchedSources) + } +} + +func TestWebhookDeliveryUnknownTokenIs404(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest("bogus-token", sonarrDownloadBody)) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", rec.Code) + } + if len(svc.ingested) != 0 { + t.Fatalf("unknown token must not ingest, got %+v", svc.ingested) + } +} + +func TestWebhookDeliveryTestEventIsAcceptedNoOp(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, + `{"eventType": "Test", "series": {"path": "C:\\testpath"}}`)) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202", rec.Code) + } + if len(svc.ingested) != 0 { + t.Fatalf("test event must not ingest, got %+v", svc.ingested) + } + if len(store.touchedSources) != 1 { + t.Fatalf("test event must stamp last_received_at, touched = %+v", store.touchedSources) + } +} + +func TestWebhookDeliveryUnknownEventTypeIsAcceptedNoOp(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, + `{"eventType": "Health", "series": {"path": "/data/tv/Show"}}`)) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202", rec.Code) + } + if len(svc.ingested) != 0 { + t.Fatalf("unknown event must not ingest, got %+v", svc.ingested) + } +} + +func TestWebhookDeliveryDisabledSourceIsAcceptedNoOp(t *testing.T) { + for name, tc := range map[string]struct { + sourceEnabled bool + settingsEnabled bool + }{ + "source disabled": {sourceEnabled: false, settingsEnabled: true}, + "autoscan disabled": {sourceEnabled: true, settingsEnabled: false}, + } { + t.Run(name, func(t *testing.T) { + store := webhookStore(webhookSource(tc.sourceEnabled), tc.settingsEnabled) + svc := &fakeAutoscanTriggerer{} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, sonarrDownloadBody)) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202", rec.Code) + } + if len(svc.ingested) != 0 { + t.Fatalf("disabled state must not ingest, got %+v", svc.ingested) + } + if len(store.touchedSources) != 1 { + t.Fatalf("valid delivery must stamp last_received_at even while disabled") + } + }) + } +} + +func TestWebhookDeliveryMalformedBodyIs400(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, "{not json")) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", rec.Code) + } + if store.webhookErrs["src-1"] == "" { + t.Fatal("parse failure must be recorded on the endpoint") + } +} + +func TestWebhookDeliveryOversizedBodyIs413(t *testing.T) { + store := webhookStore(webhookSource(true), true) + h := NewAutoscanHandler(store, &fakeAutoscanTriggerer{}) + + big := `{"eventType": "Download", "pad": "` + strings.Repeat("x", maxWebhookBodyBytes+1) + `"}` + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, big)) + + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413", rec.Code) + } +} + +func TestWebhookDeliveryDurableAcceptanceFailureIs500(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{ingestErr: context.DeadlineExceeded} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, sonarrDownloadBody)) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500 when the delivery could not be persisted", rec.Code) + } + if store.webhookErrs["src-1"] == "" { + t.Fatal("ingest failure must be recorded on the endpoint") + } +} + +func TestWebhookDeliveryPendingRetryIsAccepted(t *testing.T) { + store := webhookStore(webhookSource(true), true) + svc := &fakeAutoscanTriggerer{ingestResult: autoscan.IngestResult{Pending: true}} + h := NewAutoscanHandler(store, svc) + + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, sonarrDownloadBody)) + + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d, want 202 for a durably queued retry", rec.Code) + } +} + +// recordingLogHandler captures every log record produced during a test. +type recordingLogHandler struct { + mu sync.Mutex + entries []string +} + +func (h *recordingLogHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h *recordingLogHandler) Handle(_ context.Context, r slog.Record) error { + var b strings.Builder + b.WriteString(r.Message) + r.Attrs(func(a slog.Attr) bool { + b.WriteString(" ") + b.WriteString(a.String()) + return true + }) + h.mu.Lock() + h.entries = append(h.entries, b.String()) + h.mu.Unlock() + return nil +} +func (h *recordingLogHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recordingLogHandler) WithGroup(string) slog.Handler { return h } + +func TestWebhookDeliveryNeverLogsToken(t *testing.T) { + recorder := &recordingLogHandler{} + prev := slog.Default() + slog.SetDefault(slog.New(recorder)) + defer slog.SetDefault(prev) + + store := webhookStore(webhookSource(true), true) + // Force both the failure and success logging paths. + for _, svc := range []*fakeAutoscanTriggerer{ + {ingestErr: context.DeadlineExceeded}, + {ingestResult: autoscan.IngestResult{Enqueued: 1}}, + } { + h := NewAutoscanHandler(store, svc) + rec := httptest.NewRecorder() + h.HandleWebhookDelivery(rec, newWebhookDeliveryRequest(testWebhookToken, sonarrDownloadBody)) + } + + recorder.mu.Lock() + defer recorder.mu.Unlock() + for _, entry := range recorder.entries { + if strings.Contains(entry, testWebhookToken) { + t.Fatalf("log entry leaks the webhook token: %s", entry) + } + if strings.Contains(entry, "/data/tv/Show/Season 01") { + t.Fatalf("log entry leaks payload content: %s", entry) + } + } +} + +func TestResolveDeliveryMode(t *testing.T) { + builtin := autoscan.BuiltinArrWebhookPluginID + builtinCap := autoscan.BuiltinArrWebhookCapabilityID + cases := []struct { + name string + requested string + pluginID string + capID string + want string + wantErr bool + }{ + {"plugin defaults to poll", "", "sonarr", "arr", autoscan.DeliveryModePoll, false}, + {"builtin defaults to webhook", "", builtin, builtinCap, autoscan.DeliveryModeWebhook, false}, + {"plugin poll ok", "poll", "sonarr", "arr", autoscan.DeliveryModePoll, false}, + {"builtin webhook ok", "webhook", builtin, builtinCap, autoscan.DeliveryModeWebhook, false}, + {"plugin webhook rejected", "webhook", "sonarr", "arr", "", true}, + {"builtin poll rejected", "poll", builtin, builtinCap, "", true}, + {"garbage rejected", "carrier-pigeon", "sonarr", "arr", "", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := resolveDeliveryMode(tc.requested, tc.pluginID, tc.capID) + if tc.wantErr != (err != nil) { + t.Fatalf("err = %v, wantErr = %v", err, tc.wantErr) + } + if got != tc.want { + t.Fatalf("mode = %q, want %q", got, tc.want) + } + }) + } +} + +func TestValidateWebhookProvider(t *testing.T) { + for _, ok := range []string{"auto", "sonarr", "radarr", "", "Sonarr"} { + if err := validateWebhookProvider(map[string]string{"webhook_provider": ok}); err != nil { + t.Fatalf("provider %q must validate: %v", ok, err) + } + } + if err := validateWebhookProvider(map[string]string{"webhook_provider": "lidarr"}); err == nil { + t.Fatal("unsupported provider must be rejected") + } + if err := validateWebhookProvider(map[string]string{}); err != nil { + t.Fatalf("absent provider must validate: %v", err) + } +} + +func TestUpdatePollSourceAllowsPluginWebhookProviderKey(t *testing.T) { + existing := autoscan.Source{ + ID: "src-1", + PluginID: "example.plugin", + CapabilityID: "source", + DeliveryMode: autoscan.DeliveryModePoll, + } + var saved autoscan.Source + store := &fakeAutoscanStore{ + getSourceFn: func(string) (autoscan.Source, error) { return existing, nil }, + updateSourceFn: func(source autoscan.Source) (autoscan.Source, error) { + saved = source + return source, nil + }, + } + h := NewAutoscanHandler(store, &fakeAutoscanTriggerer{}) + body := `{"enabled":true,"delivery_mode":"poll","path_rewrites":[],"source_config":{"webhook_provider":"plugin-specific"}}` + + rec := httptest.NewRecorder() + h.HandleUpdateSource(rec, newAutoscanRequest(http.MethodPut, "/admin/autoscan/sources/src-1", body, "src-1")) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + if got := saved.SourceConfig["webhook_provider"]; got != "plugin-specific" { + t.Fatalf("plugin source config = %q, want unchanged", got) + } +} + +// --- Admin endpoint management --- + +func TestCreateSourceWebhookRequiresWebhookMode(t *testing.T) { + store := &fakeAutoscanStore{ + getSourceFn: func(id string) (autoscan.Source, error) { + return autoscan.Source{ID: id, DeliveryMode: autoscan.DeliveryModePoll}, nil + }, + } + h := NewAutoscanHandler(store, &fakeAutoscanTriggerer{}) + + rec := httptest.NewRecorder() + h.HandleCreateSourceWebhook(rec, newAutoscanRequest(http.MethodPost, "/admin/autoscan/sources/s1/webhook", "", "s1")) + + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for poll-mode source", rec.Code) + } +} + +func TestCreateSourceWebhookReturnsURL(t *testing.T) { + source := webhookSource(true) + store := &fakeAutoscanStore{ + getSourceFn: func(string) (autoscan.Source, error) { return source, nil }, + createWebhookFn: func(sourceID string) (autoscan.WebhookEndpoint, string, error) { + return autoscan.WebhookEndpoint{SourceID: sourceID, SecretSuffix: "def456"}, testWebhookToken, nil + }, + getWebhookFn: func(sourceID string) (autoscan.WebhookEndpoint, error) { + return autoscan.WebhookEndpoint{SourceID: sourceID, SecretSuffix: "def456"}, nil + }, + revealTokenFn: func(string) (string, error) { return testWebhookToken, nil }, + } + h := NewAutoscanHandler(store, &fakeAutoscanTriggerer{}) + h.SetPublicURL("https://silo.example/") + + rec := httptest.NewRecorder() + h.HandleCreateSourceWebhook(rec, newAutoscanRequest(http.MethodPost, "/admin/autoscan/sources/src-1/webhook", "", "src-1")) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + wantURL := `"webhook_url":"https://silo.example/api/v1/autoscan/webhooks/` + testWebhookToken + `"` + if !strings.Contains(body, wantURL) { + t.Fatalf("response missing %s: %s", wantURL, body) + } + if !strings.Contains(body, `"webhook_configured":true`) || !strings.Contains(body, `"webhook_secret_suffix":"def456"`) { + t.Fatalf("response missing webhook status fields: %s", body) + } +} + +func TestSourceResponsesOmitWebhookURLWhenRevealFails(t *testing.T) { + source := webhookSource(true) + store := &fakeAutoscanStore{ + listSourcesFn: func() ([]autoscan.Source, error) { return []autoscan.Source{source}, nil }, + listWebhooksFn: func() ([]autoscan.WebhookEndpoint, error) { + return []autoscan.WebhookEndpoint{{SourceID: source.ID, SecretSuffix: "def456"}}, nil + }, + revealTokenFn: func(string) (string, error) { return "", context.DeadlineExceeded }, + } + h := NewAutoscanHandler(store, &fakeAutoscanTriggerer{}) + + rec := httptest.NewRecorder() + h.HandleListSources(rec, newAutoscanRequest(http.MethodGet, "/admin/autoscan/sources", "", "")) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + body := rec.Body.String() + if strings.Contains(body, "webhook_url") { + t.Fatalf("reveal failure must omit webhook_url, got %s", body) + } + if !strings.Contains(body, `"webhook_configured":true`) { + t.Fatalf("status fields must survive reveal failure: %s", body) + } +} diff --git a/internal/api/middleware/request_logger.go b/internal/api/middleware/request_logger.go index c9407357..f4f31042 100644 --- a/internal/api/middleware/request_logger.go +++ b/internal/api/middleware/request_logger.go @@ -52,7 +52,7 @@ func RequestLogger(nodeID string) func(http.Handler) http.Handler { "component", "api", "request_id", chimw.GetReqID(r.Context()), "method", r.Method, - "path", r.URL.Path, + "path", activitylog.RedactSecretPathParams(r, r.URL.Path), "path_pattern", pathPattern, "status", wrapped.status, "duration_ms", time.Since(start).Milliseconds(), diff --git a/internal/api/router.go b/internal/api/router.go index 969ed2f6..c841ffa8 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -662,6 +662,8 @@ func NewRouter(deps Dependencies) chi.Router { if deps.TaskManager != nil { autoscanHandler.SetTriggerUpdater(deps.TaskManager) } + // Fully qualify webhook URLs when the public base URL is known. + autoscanHandler.SetPublicURL(deps.PublicURL) } if deps.PersonRepo != nil { @@ -1684,6 +1686,19 @@ func NewRouter(deps Dependencies) chi.Router { }) } + // Autoscan webhook intake: public — Sonarr/Radarr POST here without a + // Silo session; the URL's bearer token authenticates the delivery and + // maps it to its Autoscan source. Rate limited per-IP (plus the + // "autoscan_webhook" per-endpoint limit) since it is unauthenticated. + if autoscanHandler != nil { + if deps.RateLimitMW != nil { + r.With(deps.RateLimitMW.AuthEndpointHandler("autoscan_webhook")). + Post("/autoscan/webhooks/{token}", autoscanHandler.HandleWebhookDelivery) + } else { + r.Post("/autoscan/webhooks/{token}", autoscanHandler.HandleWebhookDelivery) + } + } + // Discord account-link OAuth callback: public — Discord redirects the // browser here without credentials; the one-time link-state row // authenticates the request and maps it back to the initiating @@ -2808,6 +2823,9 @@ func NewRouter(deps Dependencies) chi.Router { r.Put("/autoscan/sources/{id}", autoscanHandler.HandleUpdateSource) r.Delete("/autoscan/sources/{id}", autoscanHandler.HandleDeleteSource) r.Get("/autoscan/sources/{id}/rewrite-suggestions", autoscanHandler.HandleRewriteSuggestions) + r.Post("/autoscan/sources/{id}/webhook", autoscanHandler.HandleCreateSourceWebhook) + r.Post("/autoscan/sources/{id}/webhook/rotate", autoscanHandler.HandleRotateSourceWebhook) + r.Delete("/autoscan/sources/{id}/webhook", autoscanHandler.HandleDeleteSourceWebhook) r.Get("/autoscan/scans", autoscanHandler.HandleListScans) r.Get("/autoscan/events", autoscanHandler.HandleListEvents) r.Post("/autoscan/trigger", autoscanHandler.HandleTrigger) diff --git a/internal/autoscan/arrwebhook/parse.go b/internal/autoscan/arrwebhook/parse.go new file mode 100644 index 00000000..1170704b --- /dev/null +++ b/internal/autoscan/arrwebhook/parse.go @@ -0,0 +1,204 @@ +// Package arrwebhook parses Sonarr/Radarr webhook payloads into autoscan +// changes. It is deliberately host-side and handler-free: input is the raw +// request body, output is the provider/event classification plus the file or +// subtree paths to feed autoscan.Service.IngestChanges. Payload-derived +// content never appears in returned error messages, so they are safe to log +// and to echo in HTTP responses. +package arrwebhook + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/Silo-Server/silo-server/internal/autoscan" +) + +// Provider names accepted by Parse and stored in source_config.webhook_provider. +const ( + ProviderAuto = "auto" + ProviderSonarr = "sonarr" + ProviderRadarr = "radarr" +) + +// ErrMalformedPayload reports a body that is not valid JSON or has no +// eventType. Handlers map it to 400. +var ErrMalformedPayload = errors.New("arrwebhook: malformed payload") + +// ErrUnknownProvider reports a payload whose provider could not be inferred in +// auto mode. Handlers map it to 400. +var ErrUnknownProvider = errors.New("arrwebhook: cannot infer provider from payload") + +// ParsedWebhook is one classified delivery. Unsupported event types parse +// successfully with no changes (callers no-op them): arr adds event types over +// time and an unknown one must not make the webhook look unhealthy. +type ParsedWebhook struct { + Provider string + EventType string + Test bool + Changes []autoscan.Change +} + +// fileRef is the common shape of arr webhook file resources (episodeFile, +// movieFile, and the renamed* variants). +type fileRef struct { + Path string `json:"path"` + PreviousPath string `json:"previousPath"` +} + +type payload struct { + EventType string `json:"eventType"` + + Series *struct { + Path string `json:"path"` + } `json:"series"` + EpisodeFile *fileRef `json:"episodeFile"` + EpisodeFiles []fileRef `json:"episodeFiles"` + RenamedEpisodeFiles []fileRef `json:"renamedEpisodeFiles"` + + Movie *struct { + Path string `json:"path"` + FolderPath string `json:"folderPath"` + } `json:"movie"` + MovieFile *fileRef `json:"movieFile"` + MovieFiles []fileRef `json:"movieFiles"` + RenamedMovieFiles []fileRef `json:"renamedMovieFiles"` + + // Download/import payloads include the files replaced by an upgrade. They + // must be reconciled as vanished paths even when a separate "file delete for + // upgrade" notification is not enabled in arr. + DeletedFiles []fileRef `json:"deletedFiles"` +} + +// importEventTypes are the events that carry newly imported files. +var importEventTypes = map[string]bool{ + "Download": true, + "Import": true, + "Upgrade": true, + "DownloadComplete": true, +} + +// deleteEventTypes are the events that carry a removed file path. +var deleteEventTypes = map[string]bool{ + "EpisodeFileDelete": true, + "MovieFileDelete": true, +} + +// Parse classifies a webhook body. provider is "sonarr", "radarr", or "auto" +// (empty means auto); in auto mode the provider is inferred from the payload's +// series/movie shape. Inference failure is an error only for non-test events +// that would otherwise produce work. +func Parse(provider string, body []byte) (ParsedWebhook, error) { + var p payload + if err := json.Unmarshal(body, &p); err != nil { + return ParsedWebhook{}, ErrMalformedPayload + } + eventType := strings.TrimSpace(p.EventType) + if eventType == "" { + return ParsedWebhook{}, ErrMalformedPayload + } + + out := ParsedWebhook{EventType: eventType, Provider: resolveProvider(provider, p)} + if strings.EqualFold(eventType, "Test") { + out.Test = true + return out, nil + } + + switch { + case importEventTypes[eventType]: + out.Changes = filePaths(p, false) + if len(out.Changes) == 0 { + out.Changes = subtreeFallback(p) + } + case eventType == "Rename": + out.Changes = filePaths(p, true) + case deleteEventTypes[eventType]: + out.Changes = filePaths(p, false) + default: + // Unsupported event type: valid delivery, nothing to do. + return out, nil + } + + if out.Provider == "" { + return ParsedWebhook{}, ErrUnknownProvider + } + return out, nil +} + +// resolveProvider returns the explicit provider when set, otherwise infers +// sonarr/radarr from the payload shape ("" when inference fails). +func resolveProvider(provider string, p payload) string { + switch strings.ToLower(strings.TrimSpace(provider)) { + case ProviderSonarr: + return ProviderSonarr + case ProviderRadarr: + return ProviderRadarr + } + switch { + case p.Series != nil || p.EpisodeFile != nil || len(p.EpisodeFiles) > 0 || len(p.RenamedEpisodeFiles) > 0: + return ProviderSonarr + case p.Movie != nil || p.MovieFile != nil || len(p.MovieFiles) > 0 || len(p.RenamedMovieFiles) > 0: + return ProviderRadarr + } + return "" +} + +// filePaths collects every file path in the payload as file-scope changes, +// deduped in order. includePrevious additionally collects previousPath values +// (rename events): the previous path may no longer exist, which the host's +// resolve pipeline already handles via its vanished-path fallback. +func filePaths(p payload, includePrevious bool) []autoscan.Change { + var refs []fileRef + if p.EpisodeFile != nil { + refs = append(refs, *p.EpisodeFile) + } + refs = append(refs, p.EpisodeFiles...) + refs = append(refs, p.RenamedEpisodeFiles...) + if p.MovieFile != nil { + refs = append(refs, *p.MovieFile) + } + refs = append(refs, p.MovieFiles...) + refs = append(refs, p.RenamedMovieFiles...) + refs = append(refs, p.DeletedFiles...) + + seen := make(map[string]struct{}) + var changes []autoscan.Change + add := func(path string) { + path = strings.TrimSpace(path) + if path == "" { + return + } + if _, dup := seen[path]; dup { + return + } + seen[path] = struct{}{} + changes = append(changes, autoscan.Change{SourcePath: path, Scope: autoscan.ChangeScopeFile}) + } + for _, ref := range refs { + add(ref.Path) + if includePrevious { + add(ref.PreviousPath) + } + } + return changes +} + +// subtreeFallback maps an import event with no file details to the series or +// movie folder as one subtree-scope change. +func subtreeFallback(p payload) []autoscan.Change { + path := "" + switch { + case p.Series != nil: + path = p.Series.Path + case p.Movie != nil: + path = p.Movie.FolderPath + if strings.TrimSpace(path) == "" { + path = p.Movie.Path + } + } + path = strings.TrimSpace(path) + if path == "" { + return nil + } + return []autoscan.Change{{SourcePath: path, Scope: autoscan.ChangeScopeSubtree}} +} diff --git a/internal/autoscan/arrwebhook/parse_test.go b/internal/autoscan/arrwebhook/parse_test.go new file mode 100644 index 00000000..920db064 --- /dev/null +++ b/internal/autoscan/arrwebhook/parse_test.go @@ -0,0 +1,245 @@ +package arrwebhook + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Silo-Server/silo-server/internal/autoscan" +) + +func fixture(t *testing.T, name string) []byte { + t.Helper() + body, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + return body +} + +func paths(changes []autoscan.Change) []string { + out := make([]string, 0, len(changes)) + for _, c := range changes { + out = append(out, c.SourcePath) + } + return out +} + +func TestParseSonarrDownload(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "sonarr_download.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if parsed.Provider != ProviderSonarr || parsed.EventType != "Download" || parsed.Test { + t.Fatalf("unexpected classification: %+v", parsed) + } + want := "/data/tv/Example Show/Season 02/Example Show - S02E01 - The One That Imports.mkv" + if len(parsed.Changes) != 1 || parsed.Changes[0].SourcePath != want || parsed.Changes[0].Scope != autoscan.ChangeScopeFile { + t.Fatalf("changes = %+v", parsed.Changes) + } +} + +func TestParseSonarrRenameIncludesPreviousPaths(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "sonarr_rename.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + got := paths(parsed.Changes) + want := []string{ + "/data/tv/Example Show/Season 02/Example Show - S02E01.mkv", + "/data/tv/Example Show/Season 02/old-name-e01.mkv", + "/data/tv/Example Show/Season 02/Example Show - S02E02.mkv", + "/data/tv/Example Show/Season 02/old-name-e02.mkv", + } + if len(got) != len(want) { + t.Fatalf("paths = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("paths[%d] = %q, want %q", i, got[i], want[i]) + } + } + for _, c := range parsed.Changes { + if c.Scope != autoscan.ChangeScopeFile { + t.Fatalf("rename changes must be file scope, got %+v", c) + } + } +} + +func TestParseSonarrEpisodeFileDelete(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "sonarr_episodefile_delete.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if parsed.EventType != "EpisodeFileDelete" { + t.Fatalf("event type = %q", parsed.EventType) + } + if len(parsed.Changes) != 1 || parsed.Changes[0].SourcePath != "/data/tv/Example Show/Season 02/Example Show - S02E01.mkv" { + t.Fatalf("changes = %+v", parsed.Changes) + } +} + +func TestParseSonarrTest(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "sonarr_test.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if !parsed.Test || len(parsed.Changes) != 0 { + t.Fatalf("test event must be a no-op: %+v", parsed) + } + if parsed.Provider != ProviderSonarr { + t.Fatalf("provider = %q", parsed.Provider) + } +} + +func TestParseRadarrDownload(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "radarr_download.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if parsed.Provider != ProviderRadarr || parsed.EventType != "Download" { + t.Fatalf("unexpected classification: %+v", parsed) + } + want := "/data/movies/Example Movie (2024)/Example Movie (2024) Bluray-1080p.mkv" + if len(parsed.Changes) != 1 || parsed.Changes[0].SourcePath != want { + t.Fatalf("changes = %+v", parsed.Changes) + } +} + +func TestParseUpgradeIncludesReplacedFiles(t *testing.T) { + for name, body := range map[string]string{ + "sonarr": `{ + "eventType": "Download", + "series": {"path": "/data/tv/Show"}, + "episodeFile": {"path": "/data/tv/Show/new.mkv"}, + "deletedFiles": [{"path": "/data/tv/Show/old.mkv"}], + "isUpgrade": true + }`, + "radarr": `{ + "eventType": "Download", + "movie": {"folderPath": "/data/movies/Movie"}, + "movieFile": {"path": "/data/movies/Movie/new.mkv"}, + "deletedFiles": [{"path": "/data/movies/Movie/old.mkv"}], + "isUpgrade": true + }`, + } { + t.Run(name, func(t *testing.T) { + parsed, err := Parse(ProviderAuto, []byte(body)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + got := paths(parsed.Changes) + if len(got) != 2 || got[1] == "" || got[1] == got[0] { + t.Fatalf("upgrade paths = %v, want new and replaced file", got) + } + for _, change := range parsed.Changes { + if change.Scope != autoscan.ChangeScopeFile { + t.Fatalf("upgrade change = %+v, want file scope", change) + } + } + }) + } +} + +func TestParseRadarrRename(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "radarr_rename.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + got := paths(parsed.Changes) + want := []string{ + "/data/movies/Example Movie (2024)/Example Movie (2024) Bluray-1080p.mkv", + "/data/movies/Example Movie (2024)/old-release-name.mkv", + } + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestParseRadarrMovieFileDelete(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "radarr_moviefile_delete.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if parsed.EventType != "MovieFileDelete" || len(parsed.Changes) != 1 { + t.Fatalf("unexpected parse: %+v", parsed) + } +} + +func TestParseRadarrTest(t *testing.T) { + parsed, err := Parse(ProviderAuto, fixture(t, "radarr_test.json")) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if !parsed.Test || parsed.Provider != ProviderRadarr { + t.Fatalf("unexpected parse: %+v", parsed) + } +} + +func TestParseUnknownEventIsNoOpNotError(t *testing.T) { + parsed, err := Parse(ProviderSonarr, fixture(t, "unknown_event.json")) + if err != nil { + t.Fatalf("unknown event types must parse: %v", err) + } + if parsed.EventType != "Health" || parsed.Test || len(parsed.Changes) != 0 { + t.Fatalf("unknown event must be a no-op: %+v", parsed) + } +} + +func TestParseMalformedBody(t *testing.T) { + if _, err := Parse(ProviderAuto, []byte("{not json")); !errors.Is(err, ErrMalformedPayload) { + t.Fatalf("want ErrMalformedPayload, got %v", err) + } + if _, err := Parse(ProviderAuto, []byte(`{"noEventType": true}`)); !errors.Is(err, ErrMalformedPayload) { + t.Fatalf("missing eventType must be malformed, got %v", err) + } +} + +func TestParseAutoInferenceFailure(t *testing.T) { + // A work-producing event with neither series nor movie shape cannot be + // attributed in auto mode. + body := []byte(`{"eventType": "Download"}`) + if _, err := Parse(ProviderAuto, body); !errors.Is(err, ErrUnknownProvider) { + t.Fatalf("want ErrUnknownProvider, got %v", err) + } + // The same payload with an explicit provider parses (zero changes). + parsed, err := Parse(ProviderSonarr, body) + if err != nil { + t.Fatalf("explicit provider must parse: %v", err) + } + if len(parsed.Changes) != 0 { + t.Fatalf("changes = %+v, want none", parsed.Changes) + } +} + +func TestParseImportFallsBackToSubtree(t *testing.T) { + body := []byte(`{ + "eventType": "Download", + "series": {"path": "/data/tv/Example Show"} + }`) + parsed, err := Parse(ProviderAuto, body) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if len(parsed.Changes) != 1 || + parsed.Changes[0].SourcePath != "/data/tv/Example Show" || + parsed.Changes[0].Scope != autoscan.ChangeScopeSubtree { + t.Fatalf("changes = %+v, want one subtree fallback", parsed.Changes) + } +} + +func TestParseDedupesExactPaths(t *testing.T) { + body := []byte(`{ + "eventType": "Download", + "episodeFile": {"path": "/data/tv/S/e1.mkv"}, + "episodeFiles": [{"path": "/data/tv/S/e1.mkv"}, {"path": "/data/tv/S/e2.mkv"}] + }`) + parsed, err := Parse(ProviderAuto, body) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := paths(parsed.Changes); len(got) != 2 || got[0] != "/data/tv/S/e1.mkv" || got[1] != "/data/tv/S/e2.mkv" { + t.Fatalf("paths = %v, want deduped pair", got) + } +} diff --git a/internal/autoscan/arrwebhook/testdata/radarr_download.json b/internal/autoscan/arrwebhook/testdata/radarr_download.json new file mode 100644 index 00000000..896e0ee6 --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/radarr_download.json @@ -0,0 +1,30 @@ +{ + "movie": { + "id": 44, + "title": "Example Movie", + "year": 2024, + "folderPath": "/data/movies/Example Movie (2024)", + "tmdbId": 654321 + }, + "remoteMovie": { + "tmdbId": 654321, + "imdbId": "tt7654321", + "title": "Example Movie", + "year": 2024 + }, + "movieFile": { + "id": 771, + "relativePath": "Example Movie (2024) Bluray-1080p.mkv", + "path": "/data/movies/Example Movie (2024)/Example Movie (2024) Bluray-1080p.mkv", + "quality": "Bluray-1080p", + "qualityVersion": 1, + "releaseGroup": "GROUP", + "sceneName": "Example.Movie.2024.1080p.BluRay", + "size": 4294967296 + }, + "isUpgrade": false, + "downloadClient": "qbittorrent", + "downloadId": "def456", + "eventType": "Download", + "instanceName": "Radarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/radarr_moviefile_delete.json b/internal/autoscan/arrwebhook/testdata/radarr_moviefile_delete.json new file mode 100644 index 00000000..b3d78446 --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/radarr_moviefile_delete.json @@ -0,0 +1,20 @@ +{ + "movie": { + "id": 44, + "title": "Example Movie", + "year": 2024, + "folderPath": "/data/movies/Example Movie (2024)", + "tmdbId": 654321 + }, + "movieFile": { + "id": 771, + "relativePath": "Example Movie (2024) Bluray-1080p.mkv", + "path": "/data/movies/Example Movie (2024)/Example Movie (2024) Bluray-1080p.mkv", + "quality": "Bluray-1080p", + "qualityVersion": 1, + "size": 4294967296 + }, + "deleteReason": "deleted", + "eventType": "MovieFileDelete", + "instanceName": "Radarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/radarr_rename.json b/internal/autoscan/arrwebhook/testdata/radarr_rename.json new file mode 100644 index 00000000..776d6947 --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/radarr_rename.json @@ -0,0 +1,20 @@ +{ + "movie": { + "id": 44, + "title": "Example Movie", + "year": 2024, + "folderPath": "/data/movies/Example Movie (2024)", + "tmdbId": 654321 + }, + "renamedMovieFiles": [ + { + "id": 771, + "previousRelativePath": "old-release-name.mkv", + "previousPath": "/data/movies/Example Movie (2024)/old-release-name.mkv", + "relativePath": "Example Movie (2024) Bluray-1080p.mkv", + "path": "/data/movies/Example Movie (2024)/Example Movie (2024) Bluray-1080p.mkv" + } + ], + "eventType": "Rename", + "instanceName": "Radarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/radarr_test.json b/internal/autoscan/arrwebhook/testdata/radarr_test.json new file mode 100644 index 00000000..7a0be66f --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/radarr_test.json @@ -0,0 +1,22 @@ +{ + "movie": { + "id": 1, + "title": "Test Title", + "year": 1970, + "folderPath": "C:\\testpath", + "tmdbId": 1234 + }, + "remoteMovie": { + "tmdbId": 1234, + "imdbId": "tt1234", + "title": "Test title", + "year": 1970 + }, + "release": { + "quality": "Test Quality", + "qualityVersion": 1, + "size": 9999999 + }, + "eventType": "Test", + "instanceName": "Radarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/sonarr_download.json b/internal/autoscan/arrwebhook/testdata/sonarr_download.json new file mode 100644 index 00000000..5ee6a32c --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/sonarr_download.json @@ -0,0 +1,32 @@ +{ + "series": { + "id": 12, + "title": "Example Show", + "path": "/data/tv/Example Show", + "tvdbId": 123456, + "type": "standard" + }, + "episodes": [ + { + "id": 401, + "episodeNumber": 1, + "seasonNumber": 2, + "title": "The One That Imports" + } + ], + "episodeFile": { + "id": 991, + "relativePath": "Season 02/Example Show - S02E01 - The One That Imports.mkv", + "path": "/data/tv/Example Show/Season 02/Example Show - S02E01 - The One That Imports.mkv", + "quality": "WEBDL-1080p", + "qualityVersion": 1, + "releaseGroup": "GROUP", + "sceneName": "Example.Show.S02E01.1080p.WEB-DL", + "size": 2147483648 + }, + "isUpgrade": false, + "downloadClient": "sabnzbd", + "downloadId": "abc123", + "eventType": "Download", + "instanceName": "Sonarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/sonarr_episodefile_delete.json b/internal/autoscan/arrwebhook/testdata/sonarr_episodefile_delete.json new file mode 100644 index 00000000..a1ac8290 --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/sonarr_episodefile_delete.json @@ -0,0 +1,28 @@ +{ + "series": { + "id": 12, + "title": "Example Show", + "path": "/data/tv/Example Show", + "tvdbId": 123456, + "type": "standard" + }, + "episodes": [ + { + "id": 401, + "episodeNumber": 1, + "seasonNumber": 2, + "title": "The One That Got Deleted" + } + ], + "episodeFile": { + "id": 991, + "relativePath": "Season 02/Example Show - S02E01.mkv", + "path": "/data/tv/Example Show/Season 02/Example Show - S02E01.mkv", + "quality": "WEBDL-1080p", + "qualityVersion": 1, + "size": 2147483648 + }, + "deleteReason": "upgrade", + "eventType": "EpisodeFileDelete", + "instanceName": "Sonarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/sonarr_rename.json b/internal/autoscan/arrwebhook/testdata/sonarr_rename.json new file mode 100644 index 00000000..796ff75f --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/sonarr_rename.json @@ -0,0 +1,27 @@ +{ + "series": { + "id": 12, + "title": "Example Show", + "path": "/data/tv/Example Show", + "tvdbId": 123456, + "type": "standard" + }, + "renamedEpisodeFiles": [ + { + "id": 991, + "previousRelativePath": "Season 02/old-name-e01.mkv", + "previousPath": "/data/tv/Example Show/Season 02/old-name-e01.mkv", + "relativePath": "Season 02/Example Show - S02E01.mkv", + "path": "/data/tv/Example Show/Season 02/Example Show - S02E01.mkv" + }, + { + "id": 992, + "previousRelativePath": "Season 02/old-name-e02.mkv", + "previousPath": "/data/tv/Example Show/Season 02/old-name-e02.mkv", + "relativePath": "Season 02/Example Show - S02E02.mkv", + "path": "/data/tv/Example Show/Season 02/Example Show - S02E02.mkv" + } + ], + "eventType": "Rename", + "instanceName": "Sonarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/sonarr_test.json b/internal/autoscan/arrwebhook/testdata/sonarr_test.json new file mode 100644 index 00000000..aab6cc9b --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/sonarr_test.json @@ -0,0 +1,19 @@ +{ + "series": { + "id": 1, + "title": "Test Title", + "path": "C:\\testpath", + "tvdbId": 1234, + "type": "standard" + }, + "episodes": [ + { + "id": 123, + "episodeNumber": 1, + "seasonNumber": 1, + "title": "Test title" + } + ], + "eventType": "Test", + "instanceName": "Sonarr" +} diff --git a/internal/autoscan/arrwebhook/testdata/unknown_event.json b/internal/autoscan/arrwebhook/testdata/unknown_event.json new file mode 100644 index 00000000..f3c0ea1a --- /dev/null +++ b/internal/autoscan/arrwebhook/testdata/unknown_event.json @@ -0,0 +1,8 @@ +{ + "level": "warning", + "message": "Indexers unavailable due to failures", + "type": "IndexerStatusCheck", + "wikiUrl": "https://wiki.servarr.com/sonarr/system#indexers-are-unavailable-due-to-failures", + "eventType": "Health", + "instanceName": "Sonarr" +} diff --git a/internal/autoscan/discovery.go b/internal/autoscan/discovery.go index 4a66d429..09ef48b8 100644 --- a/internal/autoscan/discovery.go +++ b/internal/autoscan/discovery.go @@ -5,6 +5,33 @@ import ( "fmt" ) +// Built-in source identities are host-discovered scan-source entries that need +// no plugin installation. The ARR webhook identity backs webhook-mode sources: +// Sonarr/Radarr POST directly to Silo, the host parses the payload, and the +// plugin provider is never invoked for it. +const ( + BuiltinArrWebhookPluginID = "silo.autoscan.arr-webhook" + BuiltinArrWebhookCapabilityID = "arr-webhook" + builtinArrWebhookDisplayName = "Sonarr/Radarr Webhook" +) + +// BuiltinArrWebhookSource returns the host-built-in ARR webhook source +// identity offered by the Add-source picker alongside installed plugin +// capabilities. +func BuiltinArrWebhookSource() DiscoveredSource { + return DiscoveredSource{ + PluginID: BuiltinArrWebhookPluginID, + CapabilityID: BuiltinArrWebhookCapabilityID, + DisplayName: builtinArrWebhookDisplayName, + } +} + +// IsBuiltinArrWebhookIdentity reports whether (pluginID, capabilityID) is the +// built-in ARR webhook source identity. +func IsBuiltinArrWebhookIdentity(pluginID, capabilityID string) bool { + return pluginID == BuiltinArrWebhookPluginID && capabilityID == BuiltinArrWebhookCapabilityID +} + // DiscoveredSource identifies one installed scan_source.v1 capability instance, // enriched with the metadata the Add-source picker needs (plugin id + a // human-friendly display name). @@ -26,6 +53,31 @@ type ScanSourceLister interface { ListScanSources(ctx context.Context) ([]DiscoveredSource, error) } +// WithBuiltinSources wraps a ScanSourceLister so host-built-in source +// identities appear beside installed plugin capabilities. inner may be nil +// (only the builtins are returned), matching the Service's nil-lister +// tolerance. +func WithBuiltinSources(inner ScanSourceLister, builtins ...DiscoveredSource) ScanSourceLister { + return builtinSourceLister{inner: inner, builtins: builtins} +} + +type builtinSourceLister struct { + inner ScanSourceLister + builtins []DiscoveredSource +} + +func (l builtinSourceLister) ListScanSources(ctx context.Context) ([]DiscoveredSource, error) { + var out []DiscoveredSource + if l.inner != nil { + discovered, err := l.inner.ListScanSources(ctx) + if err != nil { + return nil, err + } + out = discovered + } + return append(out, l.builtins...), nil +} + // AvailableScanSource is one installed scan_source capability an operator can // create a source against (the Add-source picker list). type AvailableScanSource struct { diff --git a/internal/autoscan/discovery_test.go b/internal/autoscan/discovery_test.go index 630d22aa..9acc5613 100644 --- a/internal/autoscan/discovery_test.go +++ b/internal/autoscan/discovery_test.go @@ -31,6 +31,48 @@ func TestListAvailableScanSourcesEnumeratesInstalled(t *testing.T) { } } +func TestWithBuiltinSourcesAppendsToInner(t *testing.T) { + inner := fakeLister{sources: []DiscoveredSource{ + {PluginID: "sonarr", CapabilityID: "arr-a", DisplayName: "Sonarr"}, + }} + lister := WithBuiltinSources(inner, BuiltinArrWebhookSource()) + + discovered, err := lister.ListScanSources(context.Background()) + if err != nil { + t.Fatalf("ListScanSources: %v", err) + } + if len(discovered) != 2 { + t.Fatalf("expected 2 discovered, got %d: %+v", len(discovered), discovered) + } + if discovered[0].PluginID != "sonarr" { + t.Fatalf("plugin entries must pass through first, got %+v", discovered[0]) + } + if discovered[1] != BuiltinArrWebhookSource() { + t.Fatalf("expected builtin appended, got %+v", discovered[1]) + } +} + +func TestWithBuiltinSourcesNilInner(t *testing.T) { + lister := WithBuiltinSources(nil, BuiltinArrWebhookSource()) + + discovered, err := lister.ListScanSources(context.Background()) + if err != nil { + t.Fatalf("ListScanSources: %v", err) + } + if len(discovered) != 1 || discovered[0] != BuiltinArrWebhookSource() { + t.Fatalf("expected only builtin, got %+v", discovered) + } +} + +func TestIsBuiltinArrWebhookIdentity(t *testing.T) { + if !IsBuiltinArrWebhookIdentity(BuiltinArrWebhookPluginID, BuiltinArrWebhookCapabilityID) { + t.Fatal("builtin identity must match") + } + if IsBuiltinArrWebhookIdentity("sonarr", "arr") { + t.Fatal("plugin identity must not match builtin") + } +} + func TestListAvailableScanSourcesNilListerEmpty(t *testing.T) { svc := &Service{lister: nil} available, err := svc.ListAvailableScanSources(context.Background()) diff --git a/internal/autoscan/repository.go b/internal/autoscan/repository.go index 075a8cff..781e33f7 100644 --- a/internal/autoscan/repository.go +++ b/internal/autoscan/repository.go @@ -221,7 +221,7 @@ func (r *Repository) GetConnection(ctx context.Context, id string) (Connection, // --- Sources --- -const sourceColumns = `id, plugin_id, capability_id, connection_id, enabled, +const sourceColumns = `id, plugin_id, capability_id, connection_id, enabled, delivery_mode, poll_interval_seconds, path_rewrites, source_config, label, marker, last_run_at, last_error` func scanSource(row interface{ Scan(...any) error }) (Source, error) { @@ -229,7 +229,7 @@ func scanSource(row interface{ Scan(...any) error }) (Source, error) { var pathRewrites []byte var sourceConfig []byte if err := row.Scan(&s.ID, &s.PluginID, &s.CapabilityID, &s.ConnectionID, - &s.Enabled, &s.PollIntervalSeconds, &pathRewrites, &sourceConfig, &s.Label, &s.Marker, &s.LastRunAt, &s.LastError); err != nil { + &s.Enabled, &s.DeliveryMode, &s.PollIntervalSeconds, &pathRewrites, &sourceConfig, &s.Label, &s.Marker, &s.LastRunAt, &s.LastError); err != nil { return Source{}, err } rewrites, err := unmarshalPathRewrites(pathRewrites) @@ -313,6 +313,15 @@ func connectionIDArg(id *string) any { return nullable(*id) } +// deliveryModeArg normalizes an unset delivery mode to the poll default so +// pre-webhook callers keep today's behavior. +func deliveryModeArg(mode string) string { + if strings.TrimSpace(mode) == "" { + return DeliveryModePoll + } + return mode +} + // CreateSource inserts a new autoscan source row. One installed scan_source // capability can back many sources (e.g. one Sonarr plugin install fronting 4 // arr servers, one source per connection), so this is a plain INSERT with a @@ -329,11 +338,11 @@ func (r *Repository) CreateSource(ctx context.Context, s Source) (Source, error) } row := r.pool.QueryRow(ctx, ` INSERT INTO autoscan_sources ( - plugin_id, capability_id, connection_id, enabled, poll_interval_seconds, path_rewrites, source_config, label + plugin_id, capability_id, connection_id, enabled, delivery_mode, poll_interval_seconds, path_rewrites, source_config, label ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING `+sourceColumns, - s.PluginID, s.CapabilityID, connectionIDArg(s.ConnectionID), s.Enabled, s.PollIntervalSeconds, rewrites, sourceConfig, s.Label) + s.PluginID, s.CapabilityID, connectionIDArg(s.ConnectionID), s.Enabled, deliveryModeArg(s.DeliveryMode), s.PollIntervalSeconds, rewrites, sourceConfig, s.Label) out, err := scanSource(row) if err != nil { if connID, ok := connectionFKViolation(err, s.ConnectionID); ok { @@ -361,14 +370,15 @@ func (r *Repository) UpdateSource(ctx context.Context, s Source) (Source, error) UPDATE autoscan_sources SET connection_id = $2, enabled = $3, - poll_interval_seconds = $4, - path_rewrites = $5, - source_config = $6, - label = $7, + delivery_mode = $4, + poll_interval_seconds = $5, + path_rewrites = $6, + source_config = $7, + label = $8, updated_at = now() WHERE id = $1 RETURNING `+sourceColumns, - s.ID, connectionIDArg(s.ConnectionID), s.Enabled, s.PollIntervalSeconds, rewrites, sourceConfig, s.Label) + s.ID, connectionIDArg(s.ConnectionID), s.Enabled, deliveryModeArg(s.DeliveryMode), s.PollIntervalSeconds, rewrites, sourceConfig, s.Label) out, err := scanSource(row) if err != nil { if errors.Is(err, pgx.ErrNoRows) { @@ -511,8 +521,8 @@ func (r *Repository) RecordError(ctx context.Context, sourceID, msg string) erro } const eventColumns = `id, source_id, plugin_id, capability_id, started_at, completed_at, - duration_ms, status, changes_returned, changes_resolved, targets_claimed, scans_created, - scans_reused, scans_suppressed, error_message, marker_before, marker_after` + duration_ms, status, delivery_mode, provider_event_type, changes_returned, changes_resolved, + targets_claimed, scans_created, scans_reused, scans_suppressed, error_message, marker_before, marker_after` func scanEvent(row interface{ Scan(...any) error }) (Event, error) { var e Event @@ -526,6 +536,8 @@ func scanEvent(row interface{ Scan(...any) error }) (Event, error) { &e.CompletedAt, &e.DurationMS, &status, + &e.DeliveryMode, + &e.ProviderEventType, &e.ChangesReturned, &e.ChangesResolved, &e.TargetsClaimed, @@ -542,28 +554,43 @@ func scanEvent(row interface{ Scan(...any) error }) (Event, error) { return e, nil } +const insertEventSQL = ` + INSERT INTO autoscan_events ( + source_id, plugin_id, capability_id, started_at, completed_at, + duration_ms, status, delivery_mode, provider_event_type, error_message, marker_before + ) + VALUES ($1, $2, $3, $4, $4, 0, $5, $6, $7, $8, $9) + RETURNING id` + func (r *Repository) CreateEvent(ctx context.Context, in EventCreate) (int64, error) { started := in.StartedAt if started.IsZero() { started = time.Now() } sourceID := strings.TrimSpace(in.SourceID) - if sourceID == "" { - row := r.pool.QueryRow(ctx, ` - INSERT INTO autoscan_events ( - source_id, plugin_id, capability_id, started_at, completed_at, - duration_ms, status, error_message, marker_before - ) - VALUES ($1, $2, $3, $4, $4, 0, $5, $6, $7) - RETURNING id`, - nil, + insertArgs := func(sourceID any) []any { + return []any{ + sourceID, in.PluginID, in.CapabilityID, started, string(EventStatusRunning), + deliveryModeArg(in.DeliveryMode), + in.ProviderEventType, "", nullable(in.MarkerBefore), - ) + } + } + // Sourceless events (ad hoc triggers) and events that opt out of the + // running-event exclusion (webhook deliveries — no marker window to + // re-read, so they must never be dropped) insert without the advisory + // lock + running check. + if sourceID == "" || in.SkipRunningCheck { + var sourceArg any + if sourceID != "" { + sourceArg = sourceID + } + row := r.pool.QueryRow(ctx, insertEventSQL, insertArgs(sourceArg)...) var id int64 if err := row.Scan(&id); err != nil { return 0, fmt.Errorf("create autoscan event: %w", err) @@ -599,21 +626,7 @@ func (r *Repository) CreateEvent(ctx context.Context, in EventCreate) (int64, er return 0, fmt.Errorf("check running autoscan event: %w", err) } - row := tx.QueryRow(ctx, ` - INSERT INTO autoscan_events ( - source_id, plugin_id, capability_id, started_at, completed_at, - duration_ms, status, error_message, marker_before - ) - VALUES ($1, $2, $3, $4, $4, 0, $5, $6, $7) - RETURNING id`, - sourceID, - in.PluginID, - in.CapabilityID, - started, - string(EventStatusRunning), - "", - nullable(in.MarkerBefore), - ) + row := tx.QueryRow(ctx, insertEventSQL, insertArgs(sourceID)...) var id int64 if err := row.Scan(&id); err != nil { return 0, fmt.Errorf("create autoscan event: %w", err) diff --git a/internal/autoscan/service.go b/internal/autoscan/service.go index 0e727735..9c1fded8 100644 --- a/internal/autoscan/service.go +++ b/internal/autoscan/service.go @@ -8,6 +8,7 @@ import ( "time" "github.com/Silo-Server/silo-server/internal/scantrigger" + "github.com/google/uuid" ) const ( @@ -31,6 +32,12 @@ type Store interface { RecordError(ctx context.Context, sourceID, msg string) error CreateEvent(ctx context.Context, event EventCreate) (int64, error) FinishEvent(ctx context.Context, event EventFinish) error + CreateWebhookDelivery(ctx context.Context, in ChangeIngest) (WebhookDelivery, error) + ClaimWebhookDeliveries(ctx context.Context, workerID string, limit int) ([]WebhookDelivery, error) + CompleteWebhookDelivery(ctx context.Context, id int64, lockedBy string) error + RetryWebhookDelivery(ctx context.Context, id int64, lockedBy string, delay time.Duration, msg string) error + RecordWebhookError(ctx context.Context, sourceID, msg string) error + ClearWebhookError(ctx context.Context, sourceID string) error } // Resolver maps a Silo-native path to a concrete scan target (library folder). @@ -149,6 +156,11 @@ func (s *Service) PollOnce(ctx context.Context) error { now := time.Now() for _, src := range sources { + // Webhook sources are fed by IngestChanges when the provider POSTs to + // their endpoint; there is nothing to poll and no plugin to invoke. + if src.DeliveryMode == DeliveryModeWebhook { + continue + } // Honor the per-source poll interval as a "poll at most every N seconds" // floor: the global task fires at the default cadence, so a source with a // longer interval is skipped until enough time has elapsed. @@ -205,119 +217,342 @@ func (s *Service) PollOnce(ctx context.Context) error { continue // do NOT advance marker } - rewritten := rewriteChanges(changes, src.PathRewrites) - targets, claimed, resolvedAny, stats := s.resolveAndClaim(ctx, rewritten, ttl) - if len(targets) > maxAutoscanTargetsPerPoll { - collapsed := collapseTargetsToLibraryScans(targets) - slog.WarnContext(ctx, "autoscan: collapsed large scan target batch to library scans", "component", "autoscan", - "source_id", src.ID, - "targets", len(targets), - "collapsed_targets", len(collapsed), - "limit", maxAutoscanTargetsPerPoll, - ) - targets = collapsed - } - var enqueue EnqueueResult - if len(targets) > 0 { - var eerr error - enqueue, eerr = s.enqueueScanTargets(ctx, targets, eventID) - if eerr != nil { - s.releaseClaims(ctx, claimed) - slog.WarnContext(ctx, "autoscan: enqueue failed", "component", "autoscan", "source_id", src.ID, "err", eerr) - s.finishEvent(ctx, eventID, EventFinish{ - Status: EventStatusError, - ChangesReturned: len(changes), - ChangesResolved: stats.ChangesResolved, - TargetsClaimed: stats.TargetsClaimed, - ScansSuppressed: stats.Suppressed, - ErrorMessage: eerr.Error(), - MarkerAfter: marker, - }) - continue // do NOT advance marker - } - } - - // Advancing the marker is what tells the provider "I've consumed up to - // here". Advance it ONLY when the work it represents is genuinely done: - // - provider returned ZERO paths → nothing to do, advance normally. - // - returned paths AND ≥1 resolved → enqueued (above), advance. - // - returned paths, resolved but all - // suppressed (recently scanned / - // debounced) → work is effectively done; advance. - // - ANY resolve attempt failed INTERNALLY (resolver/database fault — - // possibly transient), whether or not other paths resolved → hold the - // marker and record the error; a later poll re-reads the same window - // once the fault clears. Advancing would silently skip the failed - // paths. Targets that DID resolve were already enqueued above; the - // re-read at worst re-scans them, which is safe. - // - returned paths AND NOTHING resolved → every path is outside Silo's - // libraries (RequestError) — a benign, expected condition for - // whole-volume filesystem watchers (e.g. CephFS), which observe - // folders that are not registered as Silo libraries. Holding here - // would pin the marker and permanently stall autoscan, so advance; - // finish the event as "unresolved" so the condition stays visible in - // poll history. - // (Some-but-not-all resolving still advances when the unresolved - // remainder is merely outside Silo's libraries.) - // - // NOTE: len(targets)==0 alone is NOT misconfiguration — paths can resolve - // yet be fully suppressed. Gate on resolvedAny, not targets. - if stats.TransientErrors > 0 { - msg := fmt.Sprintf("%d resolve attempt(s) failed internally (%d of %d path(s) resolved) — holding marker to retry", stats.TransientErrors, stats.ChangesResolved, len(changes)) - if rerr := s.store.RecordError(ctx, src.ID, msg); rerr != nil { - slog.WarnContext(ctx, "autoscan: record error failed", "component", "autoscan", "source_id", src.ID, "err", rerr) - } - s.finishEvent(ctx, eventID, EventFinish{ - Status: EventStatusError, - ChangesReturned: len(changes), - ChangesResolved: stats.ChangesResolved, - TargetsClaimed: stats.TargetsClaimed, - ScansCreated: enqueue.Created, - ScansReused: enqueue.Reused, - ScansSuppressed: stats.Suppressed, - ErrorMessage: msg, - MarkerAfter: marker, - }) - continue // do NOT advance marker - } - status := EventStatusSuccess - var statusMsg string - if len(changes) > 0 && !resolvedAny { - status = EventStatusUnresolved - statusMsg = fmt.Sprintf("returned %d path(s) but none matched a Silo library folder — advanced past them", len(changes)) - slog.WarnContext(ctx, "autoscan: returned paths matched no library folder — advancing marker", - "source_id", src.ID, "changes", len(changes)) - } - if aerr := s.store.AdvanceMarker(ctx, src.ID, next); aerr != nil { - slog.WarnContext(ctx, "autoscan: advance marker failed", "component", "autoscan", "source_id", src.ID, "err", aerr) - s.finishEvent(ctx, eventID, EventFinish{ - Status: EventStatusError, - ChangesReturned: len(changes), - ChangesResolved: stats.ChangesResolved, - TargetsClaimed: stats.TargetsClaimed, - ScansCreated: enqueue.Created, - ScansReused: enqueue.Reused, - ScansSuppressed: stats.Suppressed, - ErrorMessage: aerr.Error(), - MarkerAfter: marker, - }) - continue - } - s.finishEvent(ctx, eventID, EventFinish{ - Status: status, - ChangesReturned: len(changes), - ChangesResolved: stats.ChangesResolved, - TargetsClaimed: stats.TargetsClaimed, - ScansCreated: enqueue.Created, - ScansReused: enqueue.Reused, - ScansSuppressed: stats.Suppressed, - ErrorMessage: statusMsg, - MarkerAfter: next, + // consumeSourceChanges finishes the event and does all per-source error + // logging/recording; per-source failures never abort the poll loop. + _, _ = s.consumeSourceChanges(ctx, src, changes, consumeOptions{ + EventID: eventID, + TTL: ttl, + Marker: marker, + NextMarker: next, + AdvanceMarker: true, }) } return nil } +// consumeOptions parameterizes the shared consume path for its two callers. +// Marker/NextMarker/AdvanceMarker apply to poll-mode consumption only: webhook +// deliveries carry no marker window, never advance one, and hold nothing on +// failure (arr retries the delivery instead). +type consumeOptions struct { + EventID int64 + TTL time.Duration + Marker string // poll: the window's opening marker, held on failure + NextMarker string // poll: the provider's next marker + AdvanceMarker bool // poll: true; webhook: false +} + +// consumeResult reports what one consume pass did, for callers that surface +// the outcome (webhook delivery responses). +type consumeResult struct { + Status EventStatus + Enqueue EnqueueResult + Stats resolveStats + ResolvedAny bool +} + +// consumeSourceChanges is the shared back half of both delivery modes: it +// rewrites raw provider paths, resolves and claims scan targets, enqueues +// them, decides the event status, and finishes the event. The returned error +// reports a genuine failure (enqueue fault or transient resolve fault); the +// poll loop ignores it (already logged/recorded per source), webhook ingestion +// propagates it so the delivery can be retried by the sender. +func (s *Service) consumeSourceChanges(ctx context.Context, src Source, changes []Change, opts consumeOptions) (consumeResult, error) { + rewritten := rewriteChanges(changes, src.PathRewrites) + targets, claimed, resolvedAny, stats := s.resolveAndClaim(ctx, rewritten, opts.TTL) + result := consumeResult{Stats: stats, ResolvedAny: resolvedAny} + if len(targets) > maxAutoscanTargetsPerPoll { + collapsed := collapseTargetsToLibraryScans(targets) + slog.WarnContext(ctx, "autoscan: collapsed large scan target batch to library scans", "component", "autoscan", + "source_id", src.ID, + "targets", len(targets), + "collapsed_targets", len(collapsed), + "limit", maxAutoscanTargetsPerPoll, + ) + targets = collapsed + } + if len(targets) > 0 { + enqueue, eerr := s.enqueueScanTargets(ctx, targets, opts.EventID) + result.Enqueue = enqueue + if eerr != nil { + s.releaseClaims(ctx, claimed) + slog.WarnContext(ctx, "autoscan: enqueue failed", "component", "autoscan", "source_id", src.ID, "err", eerr) + result.Status = EventStatusError + s.finishEvent(ctx, opts.EventID, EventFinish{ + Status: EventStatusError, + ChangesReturned: len(changes), + ChangesResolved: stats.ChangesResolved, + TargetsClaimed: stats.TargetsClaimed, + ScansSuppressed: stats.Suppressed, + ErrorMessage: eerr.Error(), + MarkerAfter: opts.Marker, + }) + return result, eerr // do NOT advance marker + } + } + + // Advancing the marker is what tells the provider "I've consumed up to + // here". Advance it ONLY when the work it represents is genuinely done: + // - provider returned ZERO paths → nothing to do, advance normally. + // - returned paths AND ≥1 resolved → enqueued (above), advance. + // - returned paths, resolved but all + // suppressed (recently scanned / + // debounced) → work is effectively done; advance. + // - ANY resolve attempt failed INTERNALLY (resolver/database fault — + // possibly transient), whether or not other paths resolved → hold the + // marker and record the error; a later poll re-reads the same window + // once the fault clears. Advancing would silently skip the failed + // paths. Targets that DID resolve were already enqueued above; the + // re-read at worst re-scans them, which is safe. + // - returned paths AND NOTHING resolved → every path is outside Silo's + // libraries (RequestError) — a benign, expected condition for + // whole-volume filesystem watchers (e.g. CephFS), which observe + // folders that are not registered as Silo libraries. Holding here + // would pin the marker and permanently stall autoscan, so advance; + // finish the event as "unresolved" so the condition stays visible in + // poll history. + // (Some-but-not-all resolving still advances when the unresolved + // remainder is merely outside Silo's libraries.) + // + // Webhook deliveries have no marker: a transient resolve fault still + // finishes the event as error (the sender retries the delivery; a + // duplicate is at worst suppressed), and the unresolved case is the same + // benign "paths outside Silo's libraries" signal. + // + // NOTE: len(targets)==0 alone is NOT misconfiguration — paths can resolve + // yet be fully suppressed. Gate on resolvedAny, not targets. + if stats.TransientErrors > 0 { + msg := fmt.Sprintf("%d resolve attempt(s) failed internally (%d of %d path(s) resolved)", stats.TransientErrors, stats.ChangesResolved, len(changes)) + if opts.AdvanceMarker { + msg += " — holding marker to retry" + } + if rerr := s.store.RecordError(ctx, src.ID, msg); rerr != nil { + slog.WarnContext(ctx, "autoscan: record error failed", "component", "autoscan", "source_id", src.ID, "err", rerr) + } + result.Status = EventStatusError + s.finishEvent(ctx, opts.EventID, EventFinish{ + Status: EventStatusError, + ChangesReturned: len(changes), + ChangesResolved: stats.ChangesResolved, + TargetsClaimed: stats.TargetsClaimed, + ScansCreated: result.Enqueue.Created, + ScansReused: result.Enqueue.Reused, + ScansSuppressed: stats.Suppressed, + ErrorMessage: msg, + MarkerAfter: opts.Marker, + }) + return result, errors.New(msg) // do NOT advance marker + } + status := EventStatusSuccess + var statusMsg string + if len(changes) > 0 && !resolvedAny { + status = EventStatusUnresolved + statusMsg = fmt.Sprintf("returned %d path(s) but none matched a Silo library folder", len(changes)) + if opts.AdvanceMarker { + statusMsg += " — advanced past them" + slog.WarnContext(ctx, "autoscan: returned paths matched no library folder — advancing marker", + "source_id", src.ID, "changes", len(changes)) + } else { + slog.WarnContext(ctx, "autoscan: webhook paths matched no library folder", + "source_id", src.ID, "changes", len(changes)) + } + } + if opts.AdvanceMarker { + if aerr := s.store.AdvanceMarker(ctx, src.ID, opts.NextMarker); aerr != nil { + slog.WarnContext(ctx, "autoscan: advance marker failed", "component", "autoscan", "source_id", src.ID, "err", aerr) + result.Status = EventStatusError + s.finishEvent(ctx, opts.EventID, EventFinish{ + Status: EventStatusError, + ChangesReturned: len(changes), + ChangesResolved: stats.ChangesResolved, + TargetsClaimed: stats.TargetsClaimed, + ScansCreated: result.Enqueue.Created, + ScansReused: result.Enqueue.Reused, + ScansSuppressed: stats.Suppressed, + ErrorMessage: aerr.Error(), + MarkerAfter: opts.Marker, + }) + return result, aerr + } + } + result.Status = status + s.finishEvent(ctx, opts.EventID, EventFinish{ + Status: status, + ChangesReturned: len(changes), + ChangesResolved: stats.ChangesResolved, + TargetsClaimed: stats.TargetsClaimed, + ScansCreated: result.Enqueue.Created, + ScansReused: result.Enqueue.Reused, + ScansSuppressed: stats.Suppressed, + ErrorMessage: statusMsg, + MarkerAfter: opts.NextMarker, + }) + return result, nil +} + +// ChangeIngest is one webhook delivery's worth of changes for a source. +type ChangeIngest struct { + SourceID string + ProviderEventType string + Changes []Change + ReceivedAt time.Time +} + +// IngestResult summarizes what a webhook delivery produced. +type IngestResult struct { + Enqueued int + Suppressed int + Unresolved bool + // Pending reports that the delivery was accepted durably but its immediate + // ingest failed. The retry task will process it again inside Silo. + Pending bool +} + +var errWebhookDeliveryDisabled = errors.New("autoscan: webhook delivery disabled") + +const ( + webhookRetryBaseDelay = 5 * time.Second + webhookRetryMaxDelay = 10 * time.Minute +) + +func webhookRetryDelay(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + delay := webhookRetryBaseDelay + for i := 1; i < attempt && delay < webhookRetryMaxDelay; i++ { + delay *= 2 + if delay >= webhookRetryMaxDelay { + return webhookRetryMaxDelay + } + } + return delay +} + +// IngestChanges durably records a webhook delivery before attempting the same +// rewrite/resolve/suppress/enqueue/event pipeline as polling. A transient +// processing failure leaves the delivery queued for an internal retry and is +// surfaced as Pending rather than depending on Sonarr/Radarr to replay it. +func (s *Service) IngestChanges(ctx context.Context, in ChangeIngest) (IngestResult, error) { + delivery, err := s.store.CreateWebhookDelivery(ctx, in) + if err != nil { + return IngestResult{}, err + } + return s.processWebhookDelivery(ctx, delivery), nil +} + +// RetryPendingWebhookDeliveries claims and consumes a bounded batch of durable +// deliveries. Repository leases make concurrent nodes safe; individual ingest +// failures are rescheduled and do not abort the rest of the batch. +func (s *Service) RetryPendingWebhookDeliveries(ctx context.Context, limit int) (int, error) { + deliveries, err := s.store.ClaimWebhookDeliveries(ctx, uuid.NewString(), limit) + if err != nil { + return 0, err + } + for _, delivery := range deliveries { + s.processWebhookDelivery(ctx, delivery) + } + return len(deliveries), nil +} + +func (s *Service) processWebhookDelivery(ctx context.Context, delivery WebhookDelivery) IngestResult { + result, ingestErr := s.ingestChangesNow(ctx, ChangeIngest{ + SourceID: delivery.SourceID, + ProviderEventType: delivery.ProviderEventType, + Changes: delivery.Changes, + ReceivedAt: delivery.ReceivedAt, + }) + if ingestErr == nil { + if err := s.store.CompleteWebhookDelivery(ctx, delivery.ID, delivery.LockedBy); err != nil { + result.Pending = true + slog.WarnContext(ctx, "autoscan: complete webhook delivery failed", "component", "autoscan", "delivery_id", delivery.ID, "err", err) + } else { + if err := s.store.ClearWebhookError(ctx, delivery.SourceID); err != nil { + slog.WarnContext(ctx, "autoscan: clear webhook error failed", "component", "autoscan", "source_id", delivery.SourceID, "err", err) + } + } + return result + } + if errors.Is(ingestErr, errWebhookDeliveryDisabled) { + // A source/global disable is a pause, not a reason to discard work that + // was already accepted while enabled. Keep it pending without surfacing a + // delivery error and try again after the operator re-enables Autoscan. + result.Pending = true + if err := s.store.RetryWebhookDelivery(ctx, delivery.ID, delivery.LockedBy, time.Minute, ""); err != nil { + slog.WarnContext(ctx, "autoscan: pause webhook retry failed", "component", "autoscan", "delivery_id", delivery.ID, "err", err) + } + return result + } + + result.Pending = true + if err := s.store.RetryWebhookDelivery( + ctx, + delivery.ID, + delivery.LockedBy, + webhookRetryDelay(delivery.AttemptCount), + ingestErr.Error(), + ); err != nil { + // The durable row remains leased and becomes reclaimable when the lease + // expires, so a bookkeeping failure here still cannot lose the delivery. + slog.WarnContext(ctx, "autoscan: schedule webhook retry failed", "component", "autoscan", "delivery_id", delivery.ID, "err", err) + } + if err := s.store.RecordWebhookError(ctx, delivery.SourceID, ingestErr.Error()); err != nil { + slog.WarnContext(ctx, "autoscan: record webhook error failed", "component", "autoscan", "source_id", delivery.SourceID, "err", err) + } + return result +} + +// ingestChangesNow performs one webhook consume attempt. It is deliberately +// separate from IngestChanges so retries do not create nested delivery rows. +func (s *Service) ingestChangesNow(ctx context.Context, in ChangeIngest) (IngestResult, error) { + settings, err := s.store.GetSettings(ctx) + if err != nil { + return IngestResult{}, err + } + if !settings.Enabled { + return IngestResult{}, errWebhookDeliveryDisabled + } + src, err := s.store.GetSource(ctx, in.SourceID) + if err != nil { + return IngestResult{}, err + } + if src.DeliveryMode != DeliveryModeWebhook { + return IngestResult{}, fmt.Errorf("autoscan: source %s is not a webhook source", src.ID) + } + if !src.Enabled { + return IngestResult{}, errWebhookDeliveryDisabled + } + startedAt := in.ReceivedAt + if startedAt.IsZero() { + startedAt = time.Now() + } + // Event bookkeeping failure does not block the scan work (same resilience + // as polling): eventID 0 makes finishEvent a no-op and enqueues without an + // event link. + eventID, err := s.store.CreateEvent(ctx, EventCreate{ + SourceID: src.ID, + PluginID: src.PluginID, + CapabilityID: src.CapabilityID, + StartedAt: startedAt, + DeliveryMode: DeliveryModeWebhook, + ProviderEventType: in.ProviderEventType, + SkipRunningCheck: true, + }) + if err != nil { + slog.WarnContext(ctx, "autoscan: create webhook event failed", "component", "autoscan", "source_id", src.ID, "err", err) + eventID = 0 + } + result, cerr := s.consumeSourceChanges(ctx, src, in.Changes, consumeOptions{ + EventID: eventID, + TTL: time.Duration(settings.DebounceSeconds) * time.Second, + }) + return IngestResult{ + Enqueued: result.Enqueue.Created + result.Enqueue.Reused, + Suppressed: result.Stats.Suppressed, + Unresolved: result.Status == EventStatusUnresolved, + }, cerr +} + func (s *Service) enqueueScanTargets(ctx context.Context, targets []scantrigger.Target, eventID int64) (EnqueueResult, error) { var result EnqueueResult if len(targets) == 0 { diff --git a/internal/autoscan/service_ingest_test.go b/internal/autoscan/service_ingest_test.go new file mode 100644 index 00000000..1bb00aad --- /dev/null +++ b/internal/autoscan/service_ingest_test.go @@ -0,0 +1,220 @@ +package autoscan + +import ( + "context" + "errors" + "testing" + + "github.com/Silo-Server/silo-server/internal/scantrigger" +) + +func webhookTestStore() *fakeStore { + return &fakeStore{ + settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60}, + sources: []Source{{ + ID: "s1", + PluginID: BuiltinArrWebhookPluginID, + CapabilityID: BuiltinArrWebhookCapabilityID, + Enabled: true, + DeliveryMode: DeliveryModeWebhook, + }}, + } +} + +func TestIngestChangesEnqueuesLikePolling(t *testing.T) { + store := webhookTestStore() + prov := &fakeProvider{} + q := &recordingQueuer{} + svc := newService(store, prov, q, allowSuppressor{}) + + result, err := svc.IngestChanges(context.Background(), ChangeIngest{ + SourceID: "s1", + ProviderEventType: "Download", + Changes: []Change{ + {SourcePath: "/mnt/media/Show/Season 01/e01.mkv", Scope: ChangeScopeFile}, + {SourcePath: "/mnt/media/Show/Season 01/e02.mkv", Scope: ChangeScopeFile}, + }, + }) + if err != nil { + t.Fatalf("IngestChanges: %v", err) + } + if result.Enqueued != 2 || result.Suppressed != 0 || result.Unresolved { + t.Fatalf("unexpected result: %+v", result) + } + if len(q.enqueued) != 2 { + t.Fatalf("enqueued = %+v, want 2 file targets", q.enqueued) + } + if prov.calls != 0 { + t.Fatalf("provider must never be invoked for webhook ingestion, got %d calls", prov.calls) + } + if len(store.advanced) != 0 { + t.Fatalf("webhook ingestion must not advance markers, got %+v", store.advanced) + } + if len(store.createdEvents) != 1 { + t.Fatalf("created events = %+v, want 1", store.createdEvents) + } + created := store.createdEvents[0] + if created.DeliveryMode != DeliveryModeWebhook || created.ProviderEventType != "Download" || !created.SkipRunningCheck { + t.Fatalf("webhook event create misconfigured: %+v", created) + } + if len(store.events) != 1 || store.events[0].Status != EventStatusSuccess { + t.Fatalf("finished events = %+v, want one success", store.events) + } + if store.events[0].MarkerAfter != "" { + t.Fatalf("webhook events must not carry markers, got %q", store.events[0].MarkerAfter) + } + if len(store.deliveries) != 1 || len(store.completed) != 1 { + t.Fatalf("durable lifecycle deliveries=%d completed=%v", len(store.deliveries), store.completed) + } +} + +type failOnceResolver struct { + calls int +} + +func (r *failOnceResolver) Resolve(ctx context.Context, req scantrigger.Request) (*scantrigger.Target, error) { + r.calls++ + if r.calls == 1 { + return nil, errors.New("temporary resolver failure") + } + return (fakeResolver{}).Resolve(ctx, req) +} + +func (r *failOnceResolver) ResolveMissingSubtree(ctx context.Context, path, trigger string) (*scantrigger.Target, error) { + return (fakeResolver{}).ResolveMissingSubtree(ctx, path, trigger) +} + +func (r *failOnceResolver) ResolveVanishedPath(ctx context.Context, path, trigger string) (*scantrigger.Target, error) { + return (fakeResolver{}).ResolveVanishedPath(ctx, path, trigger) +} + +func TestRetryPendingWebhookDeliveriesRecoversTransientFailure(t *testing.T) { + store := webhookTestStore() + queue := &recordingQueuer{} + resolver := &failOnceResolver{} + svc := NewService(store, &fakeProvider{}, passthroughConnRes{}, resolver, queue, allowSuppressor{}, nil) + + result, err := svc.IngestChanges(context.Background(), ChangeIngest{ + SourceID: "s1", + ProviderEventType: "Download", + Changes: []Change{{SourcePath: "/mnt/media/Movie/movie.mkv", Scope: ChangeScopeFile}}, + }) + if err != nil || !result.Pending { + t.Fatalf("initial result = %+v, err = %v, want pending", result, err) + } + + processed, err := svc.RetryPendingWebhookDeliveries(context.Background(), 10) + if err != nil { + t.Fatalf("RetryPendingWebhookDeliveries: %v", err) + } + if processed != 1 || len(queue.enqueued) != 1 { + t.Fatalf("processed=%d enqueued=%d, want 1", processed, len(queue.enqueued)) + } + if len(store.completed) != 1 { + t.Fatalf("completed = %v, want retried delivery", store.completed) + } + if store.webhookErrors["s1"] != "" { + t.Fatalf("webhook error not cleared: %q", store.webhookErrors["s1"]) + } +} + +func TestIngestChangesRejectsPollSource(t *testing.T) { + store := webhookTestStore() + store.sources[0].DeliveryMode = DeliveryModePoll + svc := newService(store, &fakeProvider{}, &recordingQueuer{}, allowSuppressor{}) + + if _, err := svc.ingestChangesNow(context.Background(), ChangeIngest{SourceID: "s1"}); err == nil { + t.Fatal("webhook consume must reject non-webhook sources") + } +} + +func TestIngestChangesUnresolvedPathsAreBenign(t *testing.T) { + store := webhookTestStore() + svc := NewService(store, &fakeProvider{}, passthroughConnRes{}, unresolvableResolver{}, &recordingQueuer{}, allowSuppressor{}, nil) + + result, err := svc.IngestChanges(context.Background(), ChangeIngest{ + SourceID: "s1", + ProviderEventType: "Download", + Changes: []Change{{SourcePath: "/outside/library/file.mkv", Scope: ChangeScopeFile}}, + }) + if err != nil { + t.Fatalf("unresolved paths must not be an error: %v", err) + } + if !result.Unresolved || result.Enqueued != 0 { + t.Fatalf("unexpected result: %+v", result) + } + if len(store.events) != 1 || store.events[0].Status != EventStatusUnresolved { + t.Fatalf("finished events = %+v, want one unresolved", store.events) + } +} + +func TestIngestChangesTransientResolveFailureQueuesRetry(t *testing.T) { + store := webhookTestStore() + svc := NewService(store, &fakeProvider{}, passthroughConnRes{}, transientFailureResolver{}, &recordingQueuer{}, allowSuppressor{}, nil) + + result, err := svc.IngestChanges(context.Background(), ChangeIngest{ + SourceID: "s1", + ProviderEventType: "Download", + Changes: []Change{{SourcePath: "/mnt/media/Movie/movie.mkv", Scope: ChangeScopeFile}}, + }) + if err != nil { + t.Fatalf("durably accepted delivery must not depend on sender retry: %v", err) + } + if !result.Pending { + t.Fatalf("result = %+v, want pending retry", result) + } + if len(store.events) != 1 || store.events[0].Status != EventStatusError { + t.Fatalf("finished events = %+v, want one error", store.events) + } + if store.recorded["s1"] == "" { + t.Fatal("transient failure must be recorded on the source") + } + if len(store.retried) != 1 || store.retried[0].SourceID != "s1" { + t.Fatalf("durable retries = %+v, want source s1", store.retried) + } + if store.webhookErrors["s1"] == "" { + t.Fatal("transient failure must be visible on the webhook endpoint") + } +} + +func TestIngestChangesSuppressedDuplicates(t *testing.T) { + store := webhookTestStore() + q := &recordingQueuer{} + svc := NewService(store, &fakeProvider{}, passthroughConnRes{}, fakeResolver{}, q, denySuppressor{}, nil) + + result, err := svc.IngestChanges(context.Background(), ChangeIngest{ + SourceID: "s1", + ProviderEventType: "Download", + Changes: []Change{{SourcePath: "/mnt/media/Movie/movie.mkv", Scope: ChangeScopeFile}}, + }) + if err != nil { + t.Fatalf("IngestChanges: %v", err) + } + if result.Suppressed != 1 || result.Enqueued != 0 || result.Unresolved { + t.Fatalf("unexpected result: %+v", result) + } + if len(q.enqueued) != 0 { + t.Fatalf("suppressed delivery must not enqueue, got %+v", q.enqueued) + } + if len(store.events) != 1 || store.events[0].Status != EventStatusSuccess { + t.Fatalf("finished events = %+v, want one success", store.events) + } +} + +func TestPollOnceSkipsWebhookSources(t *testing.T) { + store := webhookTestStore() + prov := &fakeProvider{paths: map[string][]string{BuiltinArrWebhookCapabilityID: {"/mnt/media/Movie/movie.mkv"}}} + q := &recordingQueuer{} + svc := newService(store, prov, q, allowSuppressor{}) + + if err := svc.PollOnce(context.Background()); err != nil { + t.Fatalf("PollOnce: %v", err) + } + if prov.calls != 0 { + t.Fatalf("provider calls = %d, want 0 for webhook sources", prov.calls) + } + if len(store.createdEvents) != 0 || len(q.enqueued) != 0 { + t.Fatalf("webhook sources must be skipped by polling entirely (events=%d, enqueued=%d)", + len(store.createdEvents), len(q.enqueued)) + } +} diff --git a/internal/autoscan/service_test.go b/internal/autoscan/service_test.go index 9519dfca..d2a2a01b 100644 --- a/internal/autoscan/service_test.go +++ b/internal/autoscan/service_test.go @@ -22,6 +22,10 @@ type fakeStore struct { createdEvents []EventCreate events []EventFinish createEventErr error + deliveries []WebhookDelivery + retried []WebhookDelivery + completed []int64 + webhookErrors map[string]string } func (f *fakeStore) GetSettings(context.Context) (Settings, error) { return f.settings, nil } @@ -64,6 +68,58 @@ func (f *fakeStore) FinishEvent(_ context.Context, event EventFinish) error { f.events = append(f.events, event) return nil } +func (f *fakeStore) CreateWebhookDelivery(_ context.Context, in ChangeIngest) (WebhookDelivery, error) { + delivery := WebhookDelivery{ + ID: int64(len(f.deliveries) + 1), + SourceID: in.SourceID, + ProviderEventType: in.ProviderEventType, + Changes: append([]Change(nil), in.Changes...), + ReceivedAt: in.ReceivedAt, + AttemptCount: 1, + LockedBy: "immediate", + } + f.deliveries = append(f.deliveries, delivery) + return delivery, nil +} +func (f *fakeStore) ClaimWebhookDeliveries(_ context.Context, workerID string, limit int) ([]WebhookDelivery, error) { + if limit > len(f.retried) { + limit = len(f.retried) + } + out := append([]WebhookDelivery(nil), f.retried[:limit]...) + f.retried = f.retried[limit:] + for i := range out { + out[i].AttemptCount++ + out[i].LockedBy = workerID + } + return out, nil +} +func (f *fakeStore) CompleteWebhookDelivery(_ context.Context, id int64, _ string) error { + f.completed = append(f.completed, id) + return nil +} +func (f *fakeStore) RetryWebhookDelivery(_ context.Context, id int64, lockedBy string, _ time.Duration, _ string) error { + for _, delivery := range f.deliveries { + if delivery.ID == id { + delivery.LockedBy = lockedBy + f.retried = append(f.retried, delivery) + break + } + } + return nil +} +func (f *fakeStore) RecordWebhookError(_ context.Context, sourceID, msg string) error { + if f.webhookErrors == nil { + f.webhookErrors = map[string]string{} + } + f.webhookErrors[sourceID] = msg + return nil +} +func (f *fakeStore) ClearWebhookError(_ context.Context, sourceID string) error { + if f.webhookErrors != nil { + delete(f.webhookErrors, sourceID) + } + return nil +} // fakeProvider implements ScanSourceProvider, keyed by capability ID. type fakeProvider struct { diff --git a/internal/autoscan/types.go b/internal/autoscan/types.go index 7bace525..42468ab8 100644 --- a/internal/autoscan/types.go +++ b/internal/autoscan/types.go @@ -41,10 +41,18 @@ const ( // Change is one raw provider/source-namespace path returned by a scan_source // plugin. The host applies PathRewrites before resolving the changed path. type Change struct { - SourcePath string - Scope ChangeScope + SourcePath string `json:"source_path"` + Scope ChangeScope `json:"scope"` } +// DeliveryMode selects how a source's changes reach the host: the host polls +// the plugin (`poll`, the default), or the provider POSTs to a Silo webhook +// endpoint (`webhook`) and the plugin is never invoked. +const ( + DeliveryModePoll = "poll" + DeliveryModeWebhook = "webhook" +) + // Source ties a scan_source plugin capability to a connection plus the // host-owned scheduling/bookkeeping state. type Source struct { @@ -53,6 +61,7 @@ type Source struct { CapabilityID string ConnectionID *string // nil until an operator binds a connection Enabled bool + DeliveryMode string // DeliveryModePoll or DeliveryModeWebhook PollIntervalSeconds *int // nil => use settings default PathRewrites []PathRewrite // host-owned raw->Silo prefix rewrites SourceConfig map[string]string @@ -62,6 +71,34 @@ type Source struct { LastError *string } +// WebhookEndpoint is the admin-visible state of a source's webhook endpoint. +// The plaintext token and its lookup hash never live on this struct; creation +// and rotation return the token separately, and redisplay goes through +// RevealWebhookToken. +type WebhookEndpoint struct { + SourceID string + SecretSuffix string + CreatedAt time.Time + RotatedAt *time.Time + LastReceivedAt *time.Time + LastErrorAt *time.Time + LastErrorMessage string +} + +// WebhookDelivery is one durably accepted ARR webhook delivery waiting to be +// consumed. LockedBy is an opaque lease owner; repository completion/failure +// updates are ownership-guarded so an expired worker cannot finalize a delivery +// that another node reclaimed. +type WebhookDelivery struct { + ID int64 + SourceID string + ProviderEventType string + Changes []Change + ReceivedAt time.Time + AttemptCount int + LockedBy string +} + type EventStatus string const ( @@ -72,31 +109,40 @@ const ( ) type Event struct { - ID int64 - SourceID *string - PluginID string - CapabilityID string - StartedAt time.Time - CompletedAt time.Time - DurationMS int64 - Status EventStatus - ChangesReturned int - ChangesResolved int - TargetsClaimed int - ScansCreated int - ScansReused int - ScansSuppressed int - ErrorMessage string - MarkerBefore *string - MarkerAfter *string + ID int64 + SourceID *string + PluginID string + CapabilityID string + StartedAt time.Time + CompletedAt time.Time + DurationMS int64 + Status EventStatus + DeliveryMode string + ProviderEventType string + ChangesReturned int + ChangesResolved int + TargetsClaimed int + ScansCreated int + ScansReused int + ScansSuppressed int + ErrorMessage string + MarkerBefore *string + MarkerAfter *string } type EventCreate struct { - SourceID string - PluginID string - CapabilityID string - StartedAt time.Time - MarkerBefore string + SourceID string + PluginID string + CapabilityID string + StartedAt time.Time + MarkerBefore string + DeliveryMode string // "" defaults to DeliveryModePoll + ProviderEventType string + // SkipRunningCheck bypasses the running-event single-flight exclusion. + // Poll cycles may be skipped when an event is already running (their + // marker window is re-read next cycle); webhook deliveries carry no + // marker, so dropping one loses it forever — they always set this. + SkipRunningCheck bool } type EventFinish struct { diff --git a/internal/autoscan/webhook_delivery.go b/internal/autoscan/webhook_delivery.go new file mode 100644 index 00000000..aad6cba2 --- /dev/null +++ b/internal/autoscan/webhook_delivery.go @@ -0,0 +1,150 @@ +package autoscan + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" +) + +const webhookDeliveryLease = 5 * time.Minute + +func postgresInterval(d time.Duration) string { + return fmt.Sprintf("%f seconds", d.Seconds()) +} + +func scanWebhookDelivery(row interface{ Scan(...any) error }) (WebhookDelivery, error) { + var ( + delivery WebhookDelivery + changes []byte + ) + if err := row.Scan( + &delivery.ID, + &delivery.SourceID, + &delivery.ProviderEventType, + &changes, + &delivery.ReceivedAt, + &delivery.AttemptCount, + &delivery.LockedBy, + ); err != nil { + return WebhookDelivery{}, err + } + if err := json.Unmarshal(changes, &delivery.Changes); err != nil { + return WebhookDelivery{}, fmt.Errorf("decode autoscan webhook delivery changes: %w", err) + } + return delivery, nil +} + +// CreateWebhookDelivery durably accepts a delivery and leases it to the caller +// for an immediate ingest attempt. If the caller exits before finalizing it, a +// retry worker can reclaim it after webhookDeliveryLease. +func (r *Repository) CreateWebhookDelivery(ctx context.Context, in ChangeIngest) (WebhookDelivery, error) { + changes, err := json.Marshal(in.Changes) + if err != nil { + return WebhookDelivery{}, fmt.Errorf("encode autoscan webhook delivery changes: %w", err) + } + receivedAt := in.ReceivedAt + if receivedAt.IsZero() { + receivedAt = time.Now() + } + lockedBy := uuid.NewString() + row := r.pool.QueryRow(ctx, ` + INSERT INTO autoscan_webhook_deliveries ( + source_id, provider_event_type, changes, received_at, + attempt_count, next_attempt_at, locked_at, locked_by + ) + VALUES ($1, $2, $3, $4, 1, now(), now(), $5) + RETURNING id, source_id, provider_event_type, changes, received_at, + attempt_count, locked_by`, + in.SourceID, in.ProviderEventType, changes, receivedAt, lockedBy) + delivery, err := scanWebhookDelivery(row) + if err != nil { + return WebhookDelivery{}, fmt.Errorf("create autoscan webhook delivery: %w", err) + } + return delivery, nil +} + +// ClaimWebhookDeliveries leases due or abandoned deliveries to one worker. +// FOR UPDATE SKIP LOCKED keeps concurrent nodes from ingesting the same row. +func (r *Repository) ClaimWebhookDeliveries(ctx context.Context, workerID string, limit int) ([]WebhookDelivery, error) { + if limit <= 0 { + return []WebhookDelivery{}, nil + } + rows, err := r.pool.Query(ctx, ` + WITH due AS ( + SELECT id + FROM autoscan_webhook_deliveries + WHERE next_attempt_at <= now() + AND (locked_at IS NULL OR locked_at < now() - $2::interval) + ORDER BY next_attempt_at ASC, id ASC + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE autoscan_webhook_deliveries d + SET attempt_count = d.attempt_count + 1, + locked_at = now(), + locked_by = $3, + updated_at = now() + FROM due + WHERE d.id = due.id + RETURNING d.id, d.source_id, d.provider_event_type, d.changes, + d.received_at, d.attempt_count, d.locked_by`, + limit, postgresInterval(webhookDeliveryLease), workerID) + if err != nil { + return nil, fmt.Errorf("claim autoscan webhook deliveries: %w", err) + } + defer rows.Close() + deliveries := make([]WebhookDelivery, 0, limit) + for rows.Next() { + delivery, err := scanWebhookDelivery(rows) + if err != nil { + return nil, err + } + deliveries = append(deliveries, delivery) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate autoscan webhook deliveries: %w", err) + } + return deliveries, nil +} + +// CompleteWebhookDelivery removes a successfully consumed delivery. The lease +// owner guard prevents stale workers from deleting a row another node reclaimed. +func (r *Repository) CompleteWebhookDelivery(ctx context.Context, id int64, lockedBy string) error { + tag, err := r.pool.Exec(ctx, ` + DELETE FROM autoscan_webhook_deliveries + WHERE id = $1 AND locked_by = $2`, id, lockedBy) + if err != nil { + return fmt.Errorf("complete autoscan webhook delivery: %w", err) + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%w: webhook delivery %d lease", ErrNotFound, id) + } + return nil +} + +// RetryWebhookDelivery releases a failed delivery back to the durable queue +// after a bounded delay. The lease owner guard makes late workers harmless. +func (r *Repository) RetryWebhookDelivery(ctx context.Context, id int64, lockedBy string, delay time.Duration, msg string) error { + if delay < 0 { + delay = 0 + } + tag, err := r.pool.Exec(ctx, ` + UPDATE autoscan_webhook_deliveries + SET next_attempt_at = now() + $3::interval, + locked_at = NULL, + locked_by = '', + last_error = $4, + updated_at = now() + WHERE id = $1 AND locked_by = $2`, + id, lockedBy, postgresInterval(delay), truncateUTF8(msg, maxLastErrorLen)) + if err != nil { + return fmt.Errorf("retry autoscan webhook delivery: %w", err) + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%w: webhook delivery %d lease", ErrNotFound, id) + } + return nil +} diff --git a/internal/autoscan/webhook_endpoint.go b/internal/autoscan/webhook_endpoint.go new file mode 100644 index 00000000..547f38fc --- /dev/null +++ b/internal/autoscan/webhook_endpoint.go @@ -0,0 +1,278 @@ +package autoscan + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/Silo-Server/silo-server/internal/secret" +) + +// webhookSecretAAD binds an autoscan_webhook_endpoints secret_ref ciphertext to +// its owning source id. +func webhookSecretAAD(sourceID string) string { + return secret.RowAAD("autoscan_webhook_endpoints", "secret_ref", sourceID) +} + +// webhookSecretSuffixLen is how many trailing token characters are stored for +// UI/debug display. +const webhookSecretSuffixLen = 6 + +// newWebhookToken generates a URL-safe bearer token plus its SHA-256 lookup +// hash and display suffix. The token is 32 random bytes; plain SHA-256 is +// sufficient for lookup because the preimage is high-entropy random data. +func newWebhookToken() (token, hash, suffix string, err error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", "", "", fmt.Errorf("generate webhook token: %w", err) + } + token = base64.RawURLEncoding.EncodeToString(raw) + hash = hashWebhookToken(token) + return token, hash, token[len(token)-webhookSecretSuffixLen:], nil +} + +// hashWebhookToken maps a raw bearer token to its secret_hash lookup value. +func hashWebhookToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +const webhookEndpointColumns = `source_id, secret_suffix, created_at, rotated_at, + last_received_at, last_error_at, last_error_message` + +func scanWebhookEndpoint(row interface{ Scan(...any) error }) (WebhookEndpoint, error) { + var e WebhookEndpoint + if err := row.Scan(&e.SourceID, &e.SecretSuffix, &e.CreatedAt, &e.RotatedAt, + &e.LastReceivedAt, &e.LastErrorAt, &e.LastErrorMessage); err != nil { + return WebhookEndpoint{}, err + } + return e, nil +} + +// CreateWebhookEndpoint creates a source's webhook endpoint and returns it with +// the plaintext bearer token. When the endpoint already exists it is returned +// unchanged with an empty token (the token is only handed out on create/rotate; +// redisplay goes through RevealWebhookToken). An unknown source maps to +// ErrNotFound. +func (r *Repository) CreateWebhookEndpoint(ctx context.Context, sourceID string) (WebhookEndpoint, string, error) { + token, hash, suffix, err := newWebhookToken() + if err != nil { + return WebhookEndpoint{}, "", err + } + ref, err := r.cipher.Encrypt(token, webhookSecretAAD(sourceID)) + if err != nil { + return WebhookEndpoint{}, "", fmt.Errorf("encrypt webhook token: %w", err) + } + row := r.pool.QueryRow(ctx, ` + INSERT INTO autoscan_webhook_endpoints (source_id, secret_hash, secret_ref, secret_suffix) + VALUES ($1, $2, $3, $4) + ON CONFLICT (source_id) DO NOTHING + RETURNING `+webhookEndpointColumns, + sourceID, hash, ref, suffix) + endpoint, err := scanWebhookEndpoint(row) + if err == nil { + return endpoint, token, nil + } + if errors.Is(err, pgx.ErrNoRows) { + // Conflict: the endpoint already exists — return it, discard the + // unused freshly generated token. + existing, gerr := r.GetWebhookEndpoint(ctx, sourceID) + if gerr != nil { + return WebhookEndpoint{}, "", gerr + } + return existing, "", nil + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23503" { + return WebhookEndpoint{}, "", fmt.Errorf("%w: source %s", ErrNotFound, sourceID) + } + return WebhookEndpoint{}, "", fmt.Errorf("create autoscan webhook endpoint: %w", err) +} + +// RotateWebhookEndpoint replaces the endpoint's bearer token, invalidating the +// old URL immediately, and returns the new plaintext token. An unknown +// source/endpoint maps to ErrNotFound. +func (r *Repository) RotateWebhookEndpoint(ctx context.Context, sourceID string) (WebhookEndpoint, string, error) { + token, hash, suffix, err := newWebhookToken() + if err != nil { + return WebhookEndpoint{}, "", err + } + ref, err := r.cipher.Encrypt(token, webhookSecretAAD(sourceID)) + if err != nil { + return WebhookEndpoint{}, "", fmt.Errorf("encrypt webhook token: %w", err) + } + row := r.pool.QueryRow(ctx, ` + UPDATE autoscan_webhook_endpoints + SET secret_hash = $2, secret_ref = $3, secret_suffix = $4, rotated_at = now() + WHERE source_id = $1 + RETURNING `+webhookEndpointColumns, + sourceID, hash, ref, suffix) + endpoint, err := scanWebhookEndpoint(row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return WebhookEndpoint{}, "", fmt.Errorf("%w: webhook endpoint for source %s", ErrNotFound, sourceID) + } + return WebhookEndpoint{}, "", fmt.Errorf("rotate autoscan webhook endpoint: %w", err) + } + return endpoint, token, nil +} + +// DeleteWebhookEndpoint removes a source's webhook endpoint; future deliveries +// to the old URL return not found. An unknown endpoint maps to ErrNotFound. +func (r *Repository) DeleteWebhookEndpoint(ctx context.Context, sourceID string) error { + tag, err := r.pool.Exec(ctx, `DELETE FROM autoscan_webhook_endpoints WHERE source_id = $1`, sourceID) + if err != nil { + return fmt.Errorf("delete autoscan webhook endpoint: %w", err) + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("%w: webhook endpoint for source %s", ErrNotFound, sourceID) + } + return nil +} + +// GetWebhookEndpoint loads a source's webhook endpoint state. An unknown +// endpoint maps to ErrNotFound. +func (r *Repository) GetWebhookEndpoint(ctx context.Context, sourceID string) (WebhookEndpoint, error) { + row := r.pool.QueryRow(ctx, `SELECT `+webhookEndpointColumns+` + FROM autoscan_webhook_endpoints WHERE source_id = $1`, sourceID) + endpoint, err := scanWebhookEndpoint(row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return WebhookEndpoint{}, fmt.Errorf("%w: webhook endpoint for source %s", ErrNotFound, sourceID) + } + return WebhookEndpoint{}, fmt.Errorf("get autoscan webhook endpoint: %w", err) + } + return endpoint, nil +} + +// ListWebhookEndpoints returns every webhook endpoint (one batched query for +// admin source listings). +func (r *Repository) ListWebhookEndpoints(ctx context.Context) ([]WebhookEndpoint, error) { + rows, err := r.pool.Query(ctx, `SELECT `+webhookEndpointColumns+` + FROM autoscan_webhook_endpoints`) + if err != nil { + return nil, fmt.Errorf("list autoscan webhook endpoints: %w", err) + } + defer rows.Close() + var out []WebhookEndpoint + for rows.Next() { + endpoint, err := scanWebhookEndpoint(rows) + if err != nil { + return nil, err + } + out = append(out, endpoint) + } + return out, rows.Err() +} + +// RevealWebhookToken decrypts the stored bearer token for admin redisplay. +func (r *Repository) RevealWebhookToken(ctx context.Context, sourceID string) (string, error) { + var ref string + err := r.pool.QueryRow(ctx, `SELECT secret_ref + FROM autoscan_webhook_endpoints WHERE source_id = $1`, sourceID).Scan(&ref) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", fmt.Errorf("%w: webhook endpoint for source %s", ErrNotFound, sourceID) + } + return "", fmt.Errorf("reveal autoscan webhook token: %w", err) + } + token, err := r.cipher.Decrypt(ref, webhookSecretAAD(sourceID)) + if err != nil { + return "", fmt.Errorf("decrypt webhook token: %w", err) + } + return token, nil +} + +// ResolveWebhookToken maps a raw delivery token to its owning source and +// endpoint. Lookup is constant-shape: the token is hashed and matched against +// secret_hash; unknown, deleted, and never-existed tokens are indistinguishable +// and all map to ErrNotFound. +func (r *Repository) ResolveWebhookToken(ctx context.Context, token string) (Source, WebhookEndpoint, error) { + token = strings.TrimSpace(token) + if token == "" { + return Source{}, WebhookEndpoint{}, fmt.Errorf("%w: webhook token", ErrNotFound) + } + row := r.pool.QueryRow(ctx, ` + SELECT s.id, s.plugin_id, s.capability_id, s.connection_id, s.enabled, s.delivery_mode, + s.poll_interval_seconds, s.path_rewrites, s.source_config, s.label, s.marker, s.last_run_at, s.last_error, + e.source_id, e.secret_suffix, e.created_at, e.rotated_at, + e.last_received_at, e.last_error_at, e.last_error_message + FROM autoscan_webhook_endpoints e + JOIN autoscan_sources s ON s.id = e.source_id + WHERE e.secret_hash = $1`, + hashWebhookToken(token)) + var ( + src Source + endpoint WebhookEndpoint + pathRewrites []byte + sourceConfig []byte + ) + if err := row.Scan(&src.ID, &src.PluginID, &src.CapabilityID, &src.ConnectionID, + &src.Enabled, &src.DeliveryMode, &src.PollIntervalSeconds, &pathRewrites, &sourceConfig, + &src.Label, &src.Marker, &src.LastRunAt, &src.LastError, + &endpoint.SourceID, &endpoint.SecretSuffix, &endpoint.CreatedAt, &endpoint.RotatedAt, + &endpoint.LastReceivedAt, &endpoint.LastErrorAt, &endpoint.LastErrorMessage); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return Source{}, WebhookEndpoint{}, fmt.Errorf("%w: webhook token", ErrNotFound) + } + return Source{}, WebhookEndpoint{}, fmt.Errorf("resolve autoscan webhook token: %w", err) + } + rewrites, err := unmarshalPathRewrites(pathRewrites) + if err != nil { + return Source{}, WebhookEndpoint{}, err + } + src.PathRewrites = rewrites + config, err := unmarshalSourceConfig(sourceConfig) + if err != nil { + return Source{}, WebhookEndpoint{}, err + } + src.SourceConfig = config + return src, endpoint, nil +} + +// TouchWebhookReceived stamps the endpoint's last valid delivery time. +func (r *Repository) TouchWebhookReceived(ctx context.Context, sourceID string) error { + _, err := r.pool.Exec(ctx, ` + UPDATE autoscan_webhook_endpoints + SET last_received_at = now() + WHERE source_id = $1`, sourceID) + if err != nil { + return fmt.Errorf("touch autoscan webhook received: %w", err) + } + return nil +} + +// RecordWebhookError stores the endpoint's most recent parse/ingest failure +// (length-bounded; the caller sanitizes payload-derived content). +func (r *Repository) RecordWebhookError(ctx context.Context, sourceID, msg string) error { + _, err := r.pool.Exec(ctx, ` + UPDATE autoscan_webhook_endpoints + SET last_error_at = now(), last_error_message = $2 + WHERE source_id = $1`, sourceID, truncateUTF8(msg, maxLastErrorLen)) + if err != nil { + return fmt.Errorf("record autoscan webhook error: %w", err) + } + return nil +} + +// ClearWebhookError removes stale delivery failure state after a queued retry +// succeeds. last_received_at remains the time the provider actually delivered +// the webhook rather than being rewritten as a processing timestamp. +func (r *Repository) ClearWebhookError(ctx context.Context, sourceID string) error { + _, err := r.pool.Exec(ctx, ` + UPDATE autoscan_webhook_endpoints + SET last_error_at = NULL, last_error_message = '' + WHERE source_id = $1`, sourceID) + if err != nil { + return fmt.Errorf("clear autoscan webhook error: %w", err) + } + return nil +} diff --git a/internal/autoscan/webhook_endpoint_test.go b/internal/autoscan/webhook_endpoint_test.go new file mode 100644 index 00000000..de842428 --- /dev/null +++ b/internal/autoscan/webhook_endpoint_test.go @@ -0,0 +1,248 @@ +package autoscan + +import ( + "context" + "errors" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/secret" +) + +// newWebhookDBTest connects to SILO_TEST_DATABASE_URL (skipping when unset or +// unmigrated) and returns a repository plus a fresh webhook-mode source row. +func newWebhookDBTest(t *testing.T) (context.Context, *Repository, Source) { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + var tableName *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.autoscan_webhook_endpoints')::text`).Scan(&tableName); err != nil { + t.Fatalf("check autoscan_webhook_endpoints table: %v", err) + } + if tableName == nil || *tableName == "" { + t.Skip("test database has not applied the autoscan webhook intake migration") + } + + cipher, err := secret.New([]byte("0123456789abcdef0123456789abcdef")) + if err != nil { + t.Fatalf("new cipher: %v", err) + } + repo := NewRepository(pool, cipher) + + src, err := repo.CreateSource(ctx, Source{ + PluginID: BuiltinArrWebhookPluginID, + CapabilityID: BuiltinArrWebhookCapabilityID, + Enabled: true, + DeliveryMode: DeliveryModeWebhook, + }) + if err != nil { + t.Fatalf("create source: %v", err) + } + t.Cleanup(func() { _ = repo.DeleteSource(ctx, src.ID) }) + return ctx, repo, src +} + +func TestWebhookEndpointLifecycle(t *testing.T) { + ctx, repo, src := newWebhookDBTest(t) + + if src.DeliveryMode != DeliveryModeWebhook { + t.Fatalf("expected webhook delivery mode persisted, got %q", src.DeliveryMode) + } + + endpoint, token, err := repo.CreateWebhookEndpoint(ctx, src.ID) + if err != nil { + t.Fatalf("create endpoint: %v", err) + } + if token == "" { + t.Fatal("create must return a plaintext token") + } + if !strings.HasSuffix(token, endpoint.SecretSuffix) || len(endpoint.SecretSuffix) != webhookSecretSuffixLen { + t.Fatalf("suffix %q does not match token tail", endpoint.SecretSuffix) + } + + // Idempotent create: existing endpoint returned, no new token. + again, token2, err := repo.CreateWebhookEndpoint(ctx, src.ID) + if err != nil { + t.Fatalf("re-create endpoint: %v", err) + } + if token2 != "" || again.SecretSuffix != endpoint.SecretSuffix { + t.Fatalf("re-create must return existing endpoint with empty token, got token=%q suffix=%q", token2, again.SecretSuffix) + } + + resolvedSrc, _, err := repo.ResolveWebhookToken(ctx, token) + if err != nil { + t.Fatalf("resolve token: %v", err) + } + if resolvedSrc.ID != src.ID { + t.Fatalf("resolved source %s, want %s", resolvedSrc.ID, src.ID) + } + + revealed, err := repo.RevealWebhookToken(ctx, src.ID) + if err != nil { + t.Fatalf("reveal token: %v", err) + } + if revealed != token { + t.Fatal("revealed token must round-trip the original") + } + + rotated, newToken, err := repo.RotateWebhookEndpoint(ctx, src.ID) + if err != nil { + t.Fatalf("rotate endpoint: %v", err) + } + if newToken == "" || newToken == token { + t.Fatal("rotate must return a fresh token") + } + if rotated.RotatedAt == nil { + t.Fatal("rotate must stamp rotated_at") + } + if _, _, err := repo.ResolveWebhookToken(ctx, token); !errors.Is(err, ErrNotFound) { + t.Fatalf("old token must be invalidated, got %v", err) + } + if _, _, err := repo.ResolveWebhookToken(ctx, newToken); err != nil { + t.Fatalf("new token must resolve: %v", err) + } + + if err := repo.TouchWebhookReceived(ctx, src.ID); err != nil { + t.Fatalf("touch received: %v", err) + } + if err := repo.RecordWebhookError(ctx, src.ID, "boom"); err != nil { + t.Fatalf("record error: %v", err) + } + got, err := repo.GetWebhookEndpoint(ctx, src.ID) + if err != nil { + t.Fatalf("get endpoint: %v", err) + } + if got.LastReceivedAt == nil || got.LastErrorAt == nil || got.LastErrorMessage != "boom" { + t.Fatalf("bookkeeping fields not persisted: %+v", got) + } + + // Cascade: deleting the source removes the endpoint. + if err := repo.DeleteSource(ctx, src.ID); err != nil { + t.Fatalf("delete source: %v", err) + } + if _, err := repo.GetWebhookEndpoint(ctx, src.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("endpoint must cascade-delete with source, got %v", err) + } + if err := repo.DeleteWebhookEndpoint(ctx, src.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("delete of missing endpoint must be ErrNotFound, got %v", err) + } +} + +func TestCreateEventSkipRunningCheck(t *testing.T) { + ctx, repo, src := newWebhookDBTest(t) + + first, err := repo.CreateEvent(ctx, EventCreate{ + SourceID: src.ID, + PluginID: src.PluginID, + CapabilityID: src.CapabilityID, + DeliveryMode: DeliveryModeWebhook, + ProviderEventType: "Download", + SkipRunningCheck: true, + }) + if err != nil { + t.Fatalf("create first event: %v", err) + } + + // Poll-style creation is excluded while the first event runs... + if _, err := repo.CreateEvent(ctx, EventCreate{ + SourceID: src.ID, + PluginID: src.PluginID, + CapabilityID: src.CapabilityID, + }); !errors.Is(err, ErrPollAlreadyRunning) { + t.Fatalf("poll create must hit running exclusion, got %v", err) + } + + // ...but a webhook delivery is never dropped. + second, err := repo.CreateEvent(ctx, EventCreate{ + SourceID: src.ID, + PluginID: src.PluginID, + CapabilityID: src.CapabilityID, + DeliveryMode: DeliveryModeWebhook, + ProviderEventType: "Download", + SkipRunningCheck: true, + }) + if err != nil { + t.Fatalf("concurrent webhook event must not be dropped: %v", err) + } + + for _, id := range []int64{first, second} { + if err := repo.FinishEvent(ctx, EventFinish{ID: id, Status: EventStatusSuccess}); err != nil { + t.Fatalf("finish event %d: %v", id, err) + } + } + + events, err := repo.ListEvents(ctx, EventListFilter{SourceID: src.ID}) + if err != nil { + t.Fatalf("list events: %v", err) + } + if len(events) != 2 { + t.Fatalf("expected 2 events, got %d", len(events)) + } + for _, e := range events { + if e.Event.DeliveryMode != DeliveryModeWebhook || e.Event.ProviderEventType != "Download" { + t.Fatalf("event metadata not persisted: %+v", e.Event) + } + } +} + +func TestWebhookDeliveryDurableRetryLifecycle(t *testing.T) { + ctx, repo, src := newWebhookDBTest(t) + var tableName *string + if err := repo.pool.QueryRow(ctx, `SELECT to_regclass('public.autoscan_webhook_deliveries')::text`).Scan(&tableName); err != nil { + t.Fatalf("check autoscan_webhook_deliveries table: %v", err) + } + if tableName == nil || *tableName == "" { + t.Skip("test database has not applied the autoscan webhook delivery queue migration") + } + + delivery, err := repo.CreateWebhookDelivery(ctx, ChangeIngest{ + SourceID: src.ID, + ProviderEventType: "Download", + Changes: []Change{{SourcePath: "/data/movie.mkv", Scope: ChangeScopeFile}}, + }) + if err != nil { + t.Fatalf("create delivery: %v", err) + } + if delivery.ID == 0 || delivery.AttemptCount != 1 || delivery.LockedBy == "" { + t.Fatalf("created delivery = %+v", delivery) + } + if err := repo.RetryWebhookDelivery(ctx, delivery.ID, delivery.LockedBy, 0, "temporary"); err != nil { + t.Fatalf("schedule retry: %v", err) + } + + claimed, err := repo.ClaimWebhookDeliveries(ctx, "worker-2", 10) + if err != nil { + t.Fatalf("claim delivery: %v", err) + } + if len(claimed) != 1 || claimed[0].ID != delivery.ID || claimed[0].AttemptCount != 2 || claimed[0].LockedBy != "worker-2" { + t.Fatalf("claimed = %+v", claimed) + } + if len(claimed[0].Changes) != 1 || claimed[0].Changes[0].SourcePath != "/data/movie.mkv" { + t.Fatalf("changes did not round-trip: %+v", claimed[0].Changes) + } + if err := repo.CompleteWebhookDelivery(ctx, delivery.ID, "stale-worker"); !errors.Is(err, ErrNotFound) { + t.Fatalf("stale lease completion must be rejected, got %v", err) + } + if err := repo.CompleteWebhookDelivery(ctx, delivery.ID, "worker-2"); err != nil { + t.Fatalf("complete delivery: %v", err) + } + claimed, err = repo.ClaimWebhookDeliveries(ctx, "worker-3", 10) + if err != nil { + t.Fatalf("claim after complete: %v", err) + } + if len(claimed) != 0 { + t.Fatalf("completed delivery was reclaimed: %+v", claimed) + } +} diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index bb8b22a5..52f15057 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -9,7 +9,7 @@ import ( type Rate struct { RequestsPerSecond float64 RequestsPerMinute float64 - Burst int // max burst for token bucket (in-memory); ignored by Redis + Burst int // immediate burst allowance (token bucket or Redis second window) } // AllowResult contains the result of a rate limit check. @@ -72,6 +72,9 @@ func DefaultConfig() Config { "device_start": {RequestsPerMinute: 20, Burst: 10}, "device_lookup": {RequestsPerMinute: 60, Burst: 20}, "device_poll": {RequestsPerMinute: 120, Burst: 30}, + // Public autoscan webhook intake. Generous: arr fires one delivery + // per imported file, so season packs are legitimate bursts. + "autoscan_webhook": {RequestsPerMinute: 60, Burst: 30}, }, } } diff --git a/internal/ratelimit/redis.go b/internal/ratelimit/redis.go index 0f4a434e..5ee58174 100644 --- a/internal/ratelimit/redis.go +++ b/internal/ratelimit/redis.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "math" "time" "github.com/redis/go-redis/v9" @@ -55,6 +56,25 @@ type RedisLimiter struct { client *redis.Client } +// redisWindowLimits maps the shared rate model onto Redis fixed windows. The +// second window must honor Burst just like the in-memory token bucket; using +// only int(RequestsPerSecond) previously collapsed a 60/minute, burst-30 +// webhook limit into one request per second and rejected legitimate batches. +func redisWindowLimits(limit Rate) (secLimit, minLimit, effectiveLimit int) { + secLimit = int(math.Ceil(limit.RequestsPerSecond)) + minLimit = int(math.Ceil(limit.RequestsPerMinute)) + if limit.Burst > secLimit { + secLimit = limit.Burst + } + if secLimit <= 0 && minLimit > 0 { + secLimit = minLimit + } + if minLimit <= 0 && secLimit > 0 { + minLimit = secLimit * 60 + } + return secLimit, minLimit, secLimit +} + // NewRedisLimiter creates a new Redis-backed rate limiter. func NewRedisLimiter(client *redis.Client) *RedisLimiter { return &RedisLimiter{client: client} @@ -68,25 +88,7 @@ func (rl *RedisLimiter) Allow(ctx context.Context, key string, limit Rate) Allow secKey := fmt.Sprintf("silo:ratelimit:%s:s:%d", key, secTs) minKey := fmt.Sprintf("silo:ratelimit:%s:m:%d", key, minTs) - secLimit := int(limit.RequestsPerSecond) - minLimit := int(limit.RequestsPerMinute) - - // For sub-1 rps rates (e.g. auth endpoints: 5 req/min = 0.083 rps), - // int truncation gives secLimit=0 which makes the Lua script reject - // every request. Use the per-minute limit as the effective limit and - // skip the per-second constraint by setting it to the per-minute limit - // (the per-minute window is the real constraint). - subSecondRate := secLimit == 0 && limit.RequestsPerSecond > 0 - if subSecondRate { - secLimit = minLimit - } - - // Effective limit for response headers: use per-minute when the rate - // is defined in minutes (sub-second), per-second otherwise. - effectiveLimit := secLimit - if subSecondRate { - effectiveLimit = minLimit - } + secLimit, minLimit, effectiveLimit := redisWindowLimits(limit) result, err := rateLimitScript.Run(ctx, rl.client, []string{secKey, minKey}, @@ -124,7 +126,7 @@ func (rl *RedisLimiter) Allow(ctx context.Context, key string, limit Rate) Allow } } - remaining := effectiveLimit - minCount + remaining := minLimit - minCount if secRemaining := secLimit - secCount; secRemaining < remaining { remaining = secRemaining } diff --git a/internal/ratelimit/redis_test.go b/internal/ratelimit/redis_test.go new file mode 100644 index 00000000..473eda60 --- /dev/null +++ b/internal/ratelimit/redis_test.go @@ -0,0 +1,25 @@ +package ratelimit + +import "testing" + +func TestRedisWindowLimitsHonorBurst(t *testing.T) { + sec, minute, effective := redisWindowLimits(Rate{ + RequestsPerSecond: 1, + RequestsPerMinute: 60, + Burst: 30, + }) + if sec != 30 || minute != 60 || effective != 30 { + t.Fatalf("limits = (%d, %d, %d), want (30, 60, 30)", sec, minute, effective) + } +} + +func TestRedisWindowLimitsKeepMinuteOnlyRatesUsable(t *testing.T) { + sec, minute, _ := redisWindowLimits(Rate{ + RequestsPerSecond: 10.0 / 60.0, + RequestsPerMinute: 10, + Burst: 6, + }) + if sec != 6 || minute != 10 { + t.Fatalf("limits = (%d, %d), want burst 6 and minute 10", sec, minute) + } +} diff --git a/internal/taskmanager/tasks/autoscan_webhook_retry.go b/internal/taskmanager/tasks/autoscan_webhook_retry.go new file mode 100644 index 00000000..94068289 --- /dev/null +++ b/internal/taskmanager/tasks/autoscan_webhook_retry.go @@ -0,0 +1,58 @@ +package tasks + +import ( + "context" + "fmt" + + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +const ( + autoscanWebhookRetryIntervalMs int64 = 15 * 1000 + autoscanWebhookRetryBatch = 100 +) + +type AutoscanWebhookRetrier interface { + RetryPendingWebhookDeliveries(ctx context.Context, limit int) (int, error) +} + +// AutoscanWebhookRetryTask drains the durable webhook inbox independently of +// the scan-source poll cadence. It stays hidden because it is reliability +// plumbing rather than an operator-facing maintenance action. +type AutoscanWebhookRetryTask struct { + retrier AutoscanWebhookRetrier +} + +func NewAutoscanWebhookRetryTask(retrier AutoscanWebhookRetrier) *AutoscanWebhookRetryTask { + return &AutoscanWebhookRetryTask{retrier: retrier} +} + +func (t *AutoscanWebhookRetryTask) Key() string { return "autoscan_webhook_retry" } +func (t *AutoscanWebhookRetryTask) Name() string { return "Autoscan webhook retry" } +func (t *AutoscanWebhookRetryTask) Description() string { + return "Retry durably accepted Autoscan webhook deliveries" +} +func (t *AutoscanWebhookRetryTask) Category() taskmanager.TaskCategory { + return taskmanager.TaskCategoryLibrary +} +func (t *AutoscanWebhookRetryTask) IsHidden() bool { return true } + +func (t *AutoscanWebhookRetryTask) DefaultTriggers() []taskmanager.TriggerConfig { + return []taskmanager.TriggerConfig{{ + Type: taskmanager.TriggerTypeInterval, + IntervalMs: autoscanWebhookRetryIntervalMs, + }} +} + +func (t *AutoscanWebhookRetryTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error { + if t.retrier == nil { + return nil + } + progress.Report(0, "Checking durable webhook deliveries") + processed, err := t.retrier.RetryPendingWebhookDeliveries(ctx, autoscanWebhookRetryBatch) + if err != nil { + return fmt.Errorf("retry autoscan webhook deliveries: %w", err) + } + progress.Report(100, fmt.Sprintf("Processed %d webhook deliveries", processed)) + return nil +} diff --git a/internal/taskmanager/tasks/autoscan_webhook_retry_test.go b/internal/taskmanager/tasks/autoscan_webhook_retry_test.go new file mode 100644 index 00000000..64b9c3c6 --- /dev/null +++ b/internal/taskmanager/tasks/autoscan_webhook_retry_test.go @@ -0,0 +1,39 @@ +package tasks + +import ( + "context" + "errors" + "testing" +) + +type fakeAutoscanWebhookRetrier struct { + limit int + processed int + err error +} + +func (f *fakeAutoscanWebhookRetrier) RetryPendingWebhookDeliveries(_ context.Context, limit int) (int, error) { + f.limit = limit + return f.processed, f.err +} + +func TestAutoscanWebhookRetryTask(t *testing.T) { + retrier := &fakeAutoscanWebhookRetrier{processed: 3} + task := NewAutoscanWebhookRetryTask(retrier) + if err := task.Execute(context.Background(), noopProgressReporter{}); err != nil { + t.Fatalf("Execute: %v", err) + } + if retrier.limit != autoscanWebhookRetryBatch { + t.Fatalf("limit = %d, want %d", retrier.limit, autoscanWebhookRetryBatch) + } + if !task.IsHidden() || task.DefaultTriggers()[0].IntervalMs != autoscanWebhookRetryIntervalMs { + t.Fatalf("unexpected task configuration: hidden=%v triggers=%+v", task.IsHidden(), task.DefaultTriggers()) + } +} + +func TestAutoscanWebhookRetryTaskPropagatesClaimFailure(t *testing.T) { + task := NewAutoscanWebhookRetryTask(&fakeAutoscanWebhookRetrier{err: errors.New("database unavailable")}) + if err := task.Execute(context.Background(), noopProgressReporter{}); err == nil { + t.Fatal("Execute must report retry queue failures") + } +} diff --git a/migrations/sql/20260709143635_autoscan_webhook_intake.sql b/migrations/sql/20260709143635_autoscan_webhook_intake.sql new file mode 100644 index 00000000..6d375e43 --- /dev/null +++ b/migrations/sql/20260709143635_autoscan_webhook_intake.sql @@ -0,0 +1,44 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE autoscan_sources + ADD COLUMN delivery_mode text NOT NULL DEFAULT 'poll' + CONSTRAINT autoscan_sources_delivery_mode_check + CHECK (delivery_mode = ANY (ARRAY['poll'::text, 'webhook'::text])); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE TABLE autoscan_webhook_endpoints ( + source_id uuid PRIMARY KEY + REFERENCES autoscan_sources(id) ON DELETE CASCADE, + secret_hash text UNIQUE NOT NULL, + secret_ref text NOT NULL, + secret_suffix text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + rotated_at timestamptz, + last_received_at timestamptz, + last_error_at timestamptz, + last_error_message text NOT NULL DEFAULT '' +); +-- +goose StatementEnd + +-- +goose StatementBegin +ALTER TABLE autoscan_events + ADD COLUMN delivery_mode text NOT NULL DEFAULT 'poll', + ADD COLUMN provider_event_type text NOT NULL DEFAULT ''; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE autoscan_events + DROP COLUMN provider_event_type, + DROP COLUMN delivery_mode; +-- +goose StatementEnd + +-- +goose StatementBegin +DROP TABLE autoscan_webhook_endpoints; +-- +goose StatementEnd + +-- +goose StatementBegin +ALTER TABLE autoscan_sources + DROP COLUMN delivery_mode; +-- +goose StatementEnd diff --git a/migrations/sql/20260709175720_autoscan_webhook_delivery_queue.sql b/migrations/sql/20260709175720_autoscan_webhook_delivery_queue.sql new file mode 100644 index 00000000..a1587b5b --- /dev/null +++ b/migrations/sql/20260709175720_autoscan_webhook_delivery_queue.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE autoscan_webhook_deliveries ( + id bigserial PRIMARY KEY, + source_id uuid NOT NULL + REFERENCES autoscan_sources(id) ON DELETE CASCADE, + provider_event_type text NOT NULL, + changes jsonb NOT NULL, + received_at timestamptz NOT NULL, + attempt_count integer NOT NULL DEFAULT 1, + next_attempt_at timestamptz NOT NULL DEFAULT now(), + locked_at timestamptz, + locked_by text NOT NULL DEFAULT '', + last_error text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX idx_autoscan_webhook_deliveries_due + ON autoscan_webhook_deliveries (next_attempt_at, id); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE autoscan_webhook_deliveries; +-- +goose StatementEnd diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 65d950a4..865d29d1 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -2049,23 +2049,35 @@ export interface AutoscanPathRewrite { to: string; } +export type AutoscanDeliveryMode = "poll" | "webhook"; + +export type AutoscanWebhookProvider = "auto" | "sonarr" | "radarr"; + export interface AutoscanSource { id: string; plugin_id: string; capability_id: string; connection_id: string | null; enabled: boolean; + delivery_mode: AutoscanDeliveryMode; poll_interval_seconds: number | null; last_run_at: string | null; last_error: string | null; path_rewrites: AutoscanPathRewrite[]; source_config: Record; label: string; + webhook_configured: boolean; + webhook_url?: string; + webhook_secret_suffix?: string; + webhook_last_received_at?: string; + webhook_last_error_at?: string; + webhook_last_error_message?: string; } export interface AutoscanSourceInput { connection_id: string | null; enabled: boolean; + delivery_mode?: AutoscanDeliveryMode; poll_interval_seconds: number | null; path_rewrites?: AutoscanPathRewrite[]; source_config?: Record; @@ -2091,6 +2103,7 @@ export interface AutoscanSourceCreateInput { capability_id: string; connection_id?: string | null; enabled: boolean; + delivery_mode?: AutoscanDeliveryMode; poll_interval_seconds?: number | null; path_rewrites: AutoscanPathRewrite[]; source_config?: Record; @@ -2182,6 +2195,8 @@ export interface AutoscanEvent { completed_at: string; duration_ms: number; status: AutoscanEventStatus; + delivery_mode?: AutoscanDeliveryMode; + provider_event_type?: string; changes_returned: number; changes_resolved: number; targets_claimed: number; diff --git a/web/src/hooks/queries/useAutoscan.ts b/web/src/hooks/queries/useAutoscan.ts index 203d57de..a42150bf 100644 --- a/web/src/hooks/queries/useAutoscan.ts +++ b/web/src/hooks/queries/useAutoscan.ts @@ -198,6 +198,59 @@ export function useDeleteAutoscanSource() { }); } +// --- Webhook endpoints --- + +export function useCreateAutoscanWebhook() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + api(`/admin/autoscan/sources/${encodeURIComponent(id)}/webhook`, { + method: "POST", + }), + onSuccess: () => { + toast.success("Webhook URL created"); + queryClient.invalidateQueries({ queryKey: adminKeys.autoscanSources() }); + }, + onError: (err) => { + toast.error(err instanceof Error ? err.message : "Failed to create webhook URL"); + }, + }); +} + +export function useRotateAutoscanWebhook() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + api(`/admin/autoscan/sources/${encodeURIComponent(id)}/webhook/rotate`, { + method: "POST", + }), + onSuccess: () => { + toast.success("Webhook URL rotated — update Sonarr/Radarr with the new URL"); + queryClient.invalidateQueries({ queryKey: adminKeys.autoscanSources() }); + }, + onError: (err) => { + toast.error(err instanceof Error ? err.message : "Failed to rotate webhook URL"); + }, + }); +} + +export function useDeleteAutoscanWebhook() { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => + api(`/admin/autoscan/sources/${encodeURIComponent(id)}/webhook`, { + method: "DELETE", + }), + onSuccess: () => { + toast.success("Webhook URL deleted"); + queryClient.invalidateQueries({ queryKey: adminKeys.autoscanSources() }); + }, + onError: (err) => { + toast.error(err instanceof Error ? err.message : "Failed to delete webhook URL"); + }, + }); +} + /** * Test an arr connection. Accepts either an existing connection id, or raw * credentials (base_url + api_key_ref) / a request integration id for an diff --git a/web/src/lib/autoscanLabels.test.ts b/web/src/lib/autoscanLabels.test.ts index 95dd7c29..8b46d9d0 100644 --- a/web/src/lib/autoscanLabels.test.ts +++ b/web/src/lib/autoscanLabels.test.ts @@ -51,12 +51,14 @@ describe("resolveEventSourceName", () => { capability_id: "arr", connection_id: "conn-1", enabled: true, + delivery_mode: "poll", poll_interval_seconds: null, last_run_at: null, last_error: null, path_rewrites: [], source_config: {}, label: "", + webhook_configured: false, }; const lookups: SourceLabelLookups = { sourceByID: new Map([["src-1", source]]), diff --git a/web/src/pages/admin/autoscan/ActivityPanel.tsx b/web/src/pages/admin/autoscan/ActivityPanel.tsx index 1d2bcbbb..88972625 100644 --- a/web/src/pages/admin/autoscan/ActivityPanel.tsx +++ b/web/src/pages/admin/autoscan/ActivityPanel.tsx @@ -202,6 +202,21 @@ function ScanStatusBadge({ status }: { status: AutoscanScanStatus | ScanRun["sta ); } +/** + * Marks webhook-delivered events (with the arr event type when known) so they + * read distinctly from poll cycles. Poll events render nothing — poll is the + * default and labeling every row would be noise. + */ +function DeliveryBadge({ event }: { event: AutoscanEvent }) { + if (event.delivery_mode !== "webhook") return null; + return ( + + Webhook + {event.provider_event_type ? ` · ${event.provider_event_type}` : ""} + + ); +} + // Shared desktop table chrome so the queue, scan history, and poll log read as // one family: clipped rounded border, muted sticky-feeling header band. function DataTable({ head, children }: { head: ReactNode; children: ReactNode }) { @@ -606,6 +621,7 @@ function PollEventCard({ event, lookups }: { event: AutoscanEvent; lookups: Sour
{pollSourceName(event, lookups)} +
@@ -662,7 +678,12 @@ function PollEventTable({ - {pollSourceName(event, lookups)} + +
+ {pollSourceName(event, lookups)} + +
+ {event.error_message ? ( diff --git a/web/src/pages/admin/autoscan/SourcesPanel.tsx b/web/src/pages/admin/autoscan/SourcesPanel.tsx index ae8aec25..7a40263d 100644 --- a/web/src/pages/admin/autoscan/SourcesPanel.tsx +++ b/web/src/pages/admin/autoscan/SourcesPanel.tsx @@ -5,18 +5,22 @@ import { ChevronDown, ChevronRight, Clock, + Copy, Library as LibraryIcon, Plus, RefreshCw, Trash2, + Webhook, X, } from "lucide-react"; import { Link } from "react-router"; +import { toast } from "sonner"; import type { AutoscanPathRewrite, AutoscanRewriteSuggestions, AutoscanSource, AutoscanSourceInput, + AutoscanWebhookProvider, Library, } from "@/api/types"; import { @@ -64,7 +68,9 @@ import { useAutoscanSources, useAvailableScanSources, useCreateAutoscanSource, + useCreateAutoscanWebhook, useDeleteAutoscanSource, + useRotateAutoscanWebhook, useUpdateAutoscanSource, } from "@/hooks/queries/useAutoscan"; import { useAdminLibraries } from "@/hooks/queries/admin/libraries"; @@ -135,6 +141,22 @@ function sourceToRowEdit(source: AutoscanSource): RowEdit { }; } +const ARR_WEBHOOK_PLUGIN_ID = "silo.autoscan.arr-webhook"; +const WEBHOOK_PROVIDER_KEY = "webhook_provider"; + +function isWebhookSource(source: AutoscanSource): boolean { + return source.delivery_mode === "webhook"; +} + +function isArrWebhookPlugin(plugin: { plugin_id: string } | undefined): boolean { + return plugin?.plugin_id === ARR_WEBHOOK_PLUGIN_ID; +} + +/** Resolve a possibly relative webhook_url against the admin UI's own origin. */ +function absoluteWebhookURL(url: string): string { + return url.startsWith("/") ? `${window.location.origin}${url}` : url; +} + const CEPHFS_PLUGIN_ID = "silo.autoscan.cephfs"; const CEPHFS_CAPABILITY_ID = "cephfs"; const DEFAULT_CEPHFS_EXCLUSIONS = [ @@ -687,6 +709,147 @@ function CephFSConfigEditor({ ); } +// --------------------------------------------------------------------------- +// WebhookEndpointSection — webhook-mode replacement for the poll interval +// --------------------------------------------------------------------------- + +function WebhookEndpointSection({ + source, + provider, + onProviderChange, + isSaving, +}: { + source: AutoscanSource; + provider: AutoscanWebhookProvider; + onProviderChange: (next: AutoscanWebhookProvider) => void; + isSaving: boolean; +}) { + const createWebhook = useCreateAutoscanWebhook(); + const rotateWebhook = useRotateAutoscanWebhook(); + const [rotateOpen, setRotateOpen] = useState(false); + + const url = source.webhook_url ? absoluteWebhookURL(source.webhook_url) : ""; + + async function copyURL() { + if (!url) return; + try { + await navigator.clipboard.writeText(url); + toast.success("Webhook URL copied"); + } catch { + toast.error("Could not copy — select the URL manually"); + } + } + + return ( +
+
+ + {source.webhook_configured ? ( + <> +
+ e.currentTarget.select()} + /> + + +
+

+ Paste into Sonarr/Radarr → Settings → Connect → Webhook (On Import, On Rename, On File + Delete). +

+ + ) : ( +
+ +

+ Creates the URL Sonarr/Radarr will POST import, rename, and delete events to. +

+
+ )} +
+ +
+ + +

+ Auto infers Sonarr vs Radarr from each payload. +

+
+ + {/* Rotate confirmation */} + + + + Rotate webhook URL? + + The current URL stops working immediately. Sonarr/Radarr keep sending to the old URL + until you paste the new one into their webhook settings. + + + + Cancel + { + rotateWebhook.mutate(source.id); + setRotateOpen(false); + }} + > + Rotate + + + + +
+ ); +} + // --------------------------------------------------------------------------- // SourceRow helpers // --------------------------------------------------------------------------- @@ -814,13 +977,40 @@ function SourceRow({ }); } + function handleProviderChange(next: AutoscanWebhookProvider) { + const sourceConfig = { ...edit.sourceConfig, [WEBHOOK_PROVIDER_KEY]: next }; + setEdit((ed) => ({ ...ed, sourceConfig })); + update.mutate({ + id: source.id, + body: fullBody({ source_config: normalizeSourceConfig(sourceConfig) }), + }); + } + // Whether this source has a bound connection (server-side or pending edit). // Used to gate the Sync-from-server button, which needs a server to query. const hasEffectiveConnection = Boolean(source.connection_id) || Boolean(edit.connectionId); - // Status column - const hasError = Boolean(source.last_error); - const hasRun = Boolean(source.last_run_at); + const isWebhook = isWebhookSource(source); + + // Status column. Poll sources report last_run_at/last_error; webhook sources + // report their endpoint's delivery bookkeeping instead (deliveries do not + // stamp last_run_at). + const webhookReceivedMs = source.webhook_last_received_at + ? new Date(source.webhook_last_received_at).getTime() + : 0; + const webhookErrorMs = source.webhook_last_error_at + ? new Date(source.webhook_last_error_at).getTime() + : 0; + const hasError = isWebhook + ? webhookErrorMs > 0 && webhookErrorMs >= webhookReceivedMs + : Boolean(source.last_error); + const hasRun = isWebhook ? webhookReceivedMs > 0 : Boolean(source.last_run_at); + const statusErrorMessage = isWebhook + ? source.webhook_last_error_message || source.last_error || "Delivery failed" + : (source.last_error ?? ""); + const statusTimestamp = isWebhook + ? (source.webhook_last_received_at ?? null) + : source.last_run_at; const connectionSelectClass = layout === "card" ? "!w-full min-w-0 max-w-full" : "w-[200px]"; const statusMessageClass = layout === "card" @@ -845,7 +1035,15 @@ function SourceRow({ const sourceIdentity = (
-

{resolvedLabel.name}

+

+ {resolvedLabel.name} + {isWebhook && ( + + + Webhook + + )} +

{resolvedLabel.detail}

Error

-

- {source.last_error} +

+ {statusErrorMessage}

@@ -876,39 +1074,23 @@ function SourceRow({

OK

- {formatRelativeTime(source.last_run_at)} + {formatRelativeTime(statusTimestamp)}

) : ( - Not run yet + + {isWebhook ? "No deliveries yet" : "Not run yet"} + ); - const connectionControl = - source.connection_id === null && !edit.connectionId ? ( -
- - - No connection - -
- ) : ( - - ); + + No connection + + + ) : ( + + ); - const intervalSettings = ( + // Webhook payload paths are arr-side paths and still need host-side + // rewrites in Docker/mount-mismatch setups, so the rewrite editor stays + // visible in both modes. + const rewriteEditor = ( + setEdit((ed) => ({ ...ed, rewrites: next }))} + onSave={handleRewriteSave} + isSaving={update.isPending} + /> + ); + + const intervalSettings = isWebhook ? ( + <> + + {rewriteEditor} + + ) : ( <>
Must be a positive integer.

)}

{intervalHelp}

- setEdit((ed) => ({ ...ed, rewrites: next }))} - onSave={handleRewriteSave} - isSaving={update.isPending} - /> + {rewriteEditor} {isCephFSSource(source) && (
-
- - {connectionControl} -
+ {!isWebhook && ( +
+ + {connectionControl} +
+ )}

Enabled

- Poll this source for changes + {isWebhook ? "Accept webhook deliveries" : "Poll this source for changes"}

- +
{statusNode}
- + {intervalSettings}
@@ -1097,6 +1325,7 @@ function AddSourceDialog({ (p) => pluginKey(p.plugin_id, p.capability_id) === form.pluginKey, ); const selectedIsCephFS = isCephFSPlugin(selectedPlugin); + const selectedIsWebhook = isArrWebhookPlugin(selectedPlugin); function close() { setForm(BLANK_ADD_SOURCE); @@ -1105,6 +1334,20 @@ function AddSourceDialog({ function handleSubmit() { if (!selectedPlugin) return; + if (selectedIsWebhook) { + createSource.mutate( + { + plugin_id: selectedPlugin.plugin_id, + capability_id: selectedPlugin.capability_id, + enabled: false, + delivery_mode: "webhook", + path_rewrites: [], + source_config: { [WEBHOOK_PROVIDER_KEY]: "auto" }, + }, + { onSuccess: close }, + ); + return; + } const connectionId = form.connectionId && form.connectionId !== "__none__" ? form.connectionId : null; const raw = form.intervalStr.trim(); @@ -1124,6 +1367,7 @@ function AddSourceDialog({ } const intervalInvalid = + !selectedIsWebhook && form.intervalStr.trim() !== "" && (!Number.isInteger(Number(form.intervalStr)) || Number(form.intervalStr) < 1); const canSubmit = Boolean(selectedPlugin) && !intervalInvalid && !createSource.isPending; @@ -1184,50 +1428,67 @@ function AddSourceDialog({
-
- - -

- Optional — bind one if the source needs to reach a server (e.g. Sonarr/Radarr). - Sources that read locally need no connection. -

-
- -
- -
- setForm((f) => ({ ...f, intervalStr: e.target.value }))} - /> - sec + {selectedIsWebhook && ( +
+

+ + Direct webhook delivery +

+

+ No arr API key or connection needed. After creating the source, generate its + webhook URL and paste it into Sonarr/Radarr → Settings → Connect → Webhook. +

- {intervalInvalid && ( -

Must be a positive integer.

- )} -

- Optional — leave blank to use the global default poll interval. -

-
+ )} + + {!selectedIsWebhook && ( +
+ + +

+ Optional — bind one if the source needs to reach a server (e.g. Sonarr/Radarr). + Sources that read locally need no connection. +

+
+ )} + + {!selectedIsWebhook && ( +
+ +
+ setForm((f) => ({ ...f, intervalStr: e.target.value }))} + /> + sec +
+ {intervalInvalid && ( +

Must be a positive integer.

+ )} +

+ Optional — leave blank to use the global default poll interval. +

+
+ )} {selectedIsCephFS && (