* docs(autoscan): add arr webhook intake spec and implementation plan Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(autoscan): add webhook intake schema migration Adds delivery_mode to autoscan_sources, the autoscan_webhook_endpoints table, and delivery_mode/provider_event_type on autoscan_events. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(autoscan): add built-in arr-webhook source identity Host-discovered scan-source entry so webhook-mode sources need no plugin installation; composite lister appends it to plugin discovery. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(autoscan): persist delivery mode, webhook endpoints, event metadata Sources carry delivery_mode; autoscan_webhook_endpoints CRUD with SHA-256 token lookup and AAD-bound encrypted redisplay; events record delivery_mode/provider_event_type; CreateEvent gains SkipRunningCheck so webhook deliveries are never dropped by the poll exclusion. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(autoscan): share the consume path and add webhook IngestChanges Extracts consumeSourceChanges from PollOnce (marker semantics preserved, existing poll tests unchanged); PollOnce skips webhook sources; IngestChanges feeds deliveries through the shared pipeline without markers and without the running-event exclusion. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(autoscan): add Sonarr/Radarr webhook payload parser Host-side arrwebhook package: provider inference, import/rename/delete path extraction with vanished-path-friendly previous paths, subtree fallback, exact-path dedupe, and no-op unknown events. Fixture-backed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(autoscan): add public webhook delivery route and admin endpoint management Public POST /api/v1/autoscan/webhooks/{token} with per-IP rate limiting, 256KiB body cap, 202-for-noop semantics, and token/body kept out of logs; admin create/rotate/delete endpoint routes; source responses carry delivery mode + webhook status/URL; create/update validate delivery mode against source identity. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(web): add webhook delivery mode to Autoscan admin UI Webhook sources get a generate/copy/rotate webhook URL section, provider selector, delivery status, and a connection-free Add-source flow; activity rows badge webhook deliveries with the arr event type. Path rewrites stay editable in both modes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(api): redact secret path params from request and activity logs The request logger and activity-log middleware recorded raw URLs, so bearer credentials in secret path segments (autoscan webhook {token}, webhook-sync {secret}) were persisted to app logs and activity_log. Redact the secret segment via the chi route params in both sinks. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(autoscan): make webhook delivery reliable --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
148 lines
4.1 KiB
Go
148 lines
4.1 KiB
Go
package ratelimit
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"math"
|
|
"time"
|
|
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
// Lua script: check-then-increment for two window counters.
|
|
// KEYS[1] = per-second key, KEYS[2] = per-minute key
|
|
// ARGV[1] = per-second limit, ARGV[2] = per-minute limit
|
|
// Returns: {allowed(0/1), sec_count, sec_limit, min_count, min_limit, sec_ttl}
|
|
var rateLimitScript = redis.NewScript(`
|
|
local sec_key = KEYS[1]
|
|
local min_key = KEYS[2]
|
|
local sec_limit = tonumber(ARGV[1])
|
|
local min_limit = tonumber(ARGV[2])
|
|
|
|
-- Check current counts before incrementing
|
|
local sec_count = tonumber(redis.call('GET', sec_key) or "0")
|
|
local min_count = tonumber(redis.call('GET', min_key) or "0")
|
|
|
|
-- Deny if either limit is exceeded
|
|
if sec_count >= sec_limit then
|
|
local sec_ttl = redis.call('TTL', sec_key)
|
|
if sec_ttl < 0 then sec_ttl = 1 end
|
|
return {0, sec_count, sec_limit, min_count, min_limit, sec_ttl}
|
|
end
|
|
if min_count >= min_limit then
|
|
local sec_ttl = redis.call('TTL', sec_key)
|
|
if sec_ttl < 0 then sec_ttl = 1 end
|
|
return {0, sec_count, sec_limit, min_count, min_limit, sec_ttl}
|
|
end
|
|
|
|
-- Allowed: increment both counters
|
|
sec_count = redis.call('INCR', sec_key)
|
|
if sec_count == 1 then
|
|
redis.call('EXPIRE', sec_key, 2)
|
|
end
|
|
|
|
min_count = redis.call('INCR', min_key)
|
|
if min_count == 1 then
|
|
redis.call('EXPIRE', min_key, 120)
|
|
end
|
|
|
|
local sec_ttl = redis.call('TTL', sec_key)
|
|
return {1, sec_count, sec_limit, min_count, min_limit, sec_ttl}
|
|
`)
|
|
|
|
// RedisLimiter is a Redis-backed rate limiter using fixed window counters.
|
|
type RedisLimiter struct {
|
|
client *redis.Client
|
|
}
|
|
|
|
// redisWindowLimits maps the shared rate model onto Redis fixed windows. The
|
|
// second window must honor Burst just like the in-memory token bucket; using
|
|
// only int(RequestsPerSecond) previously collapsed a 60/minute, burst-30
|
|
// webhook limit into one request per second and rejected legitimate batches.
|
|
func redisWindowLimits(limit Rate) (secLimit, minLimit, effectiveLimit int) {
|
|
secLimit = int(math.Ceil(limit.RequestsPerSecond))
|
|
minLimit = int(math.Ceil(limit.RequestsPerMinute))
|
|
if limit.Burst > secLimit {
|
|
secLimit = limit.Burst
|
|
}
|
|
if secLimit <= 0 && minLimit > 0 {
|
|
secLimit = minLimit
|
|
}
|
|
if minLimit <= 0 && secLimit > 0 {
|
|
minLimit = secLimit * 60
|
|
}
|
|
return secLimit, minLimit, secLimit
|
|
}
|
|
|
|
// NewRedisLimiter creates a new Redis-backed rate limiter.
|
|
func NewRedisLimiter(client *redis.Client) *RedisLimiter {
|
|
return &RedisLimiter{client: client}
|
|
}
|
|
|
|
func (rl *RedisLimiter) Allow(ctx context.Context, key string, limit Rate) AllowResult {
|
|
now := time.Now()
|
|
secTs := now.Unix()
|
|
minTs := now.Unix() / 60
|
|
|
|
secKey := fmt.Sprintf("silo:ratelimit:%s:s:%d", key, secTs)
|
|
minKey := fmt.Sprintf("silo:ratelimit:%s:m:%d", key, minTs)
|
|
|
|
secLimit, minLimit, effectiveLimit := redisWindowLimits(limit)
|
|
|
|
result, err := rateLimitScript.Run(ctx, rl.client,
|
|
[]string{secKey, minKey},
|
|
secLimit, minLimit,
|
|
).Int64Slice()
|
|
|
|
if err != nil {
|
|
// Fail-open: allow request if Redis is unreachable
|
|
slog.WarnContext(ctx, "rate limit Redis error, allowing request", "component", "ratelimit", "error", err, "key", key)
|
|
return AllowResult{
|
|
Allowed: true,
|
|
Limit: effectiveLimit,
|
|
Remaining: -1, // unknown -- signals fail-open to callers
|
|
ResetAt: now.Add(time.Second).Truncate(time.Second),
|
|
}
|
|
}
|
|
|
|
allowed := result[0] == 1
|
|
secCount := int(result[1])
|
|
minCount := int(result[3])
|
|
secTTL := time.Duration(result[5]) * time.Second
|
|
|
|
if !allowed {
|
|
// Determine which limit was hit for RetryAfter
|
|
retryAfter := secTTL
|
|
if secCount < secLimit && minCount >= minLimit {
|
|
retryAfter = time.Duration(60-(now.Unix()%60)) * time.Second
|
|
}
|
|
return AllowResult{
|
|
Allowed: false,
|
|
RetryAfter: retryAfter,
|
|
Limit: effectiveLimit,
|
|
Remaining: 0,
|
|
ResetAt: now.Add(retryAfter),
|
|
}
|
|
}
|
|
|
|
remaining := minLimit - minCount
|
|
if secRemaining := secLimit - secCount; secRemaining < remaining {
|
|
remaining = secRemaining
|
|
}
|
|
if remaining < 0 {
|
|
remaining = 0
|
|
}
|
|
|
|
return AllowResult{
|
|
Allowed: true,
|
|
Limit: effectiveLimit,
|
|
Remaining: remaining,
|
|
ResetAt: now.Add(time.Second).Truncate(time.Second),
|
|
}
|
|
}
|
|
|
|
func (rl *RedisLimiter) Close() {
|
|
// Redis client lifecycle managed externally
|
|
}
|