Files
silo-server/internal/cache/redis.go
T
0163df3683 [codex] Add IntroDB marker integration and dialogue-aware Chromaprint refinement (#57)
* docs(markers): design + implementation plans for multi-source markers & TheIntroDB contribution

* fix(markers): TheIntroDB read-path correctness (TVDB, real confidence, best candidate)

Honor TVDB ids in /media lookups (previously dropped — anime/TheTVDB-first
libraries got no markers), decode and use the real per-segment confidence and
submission_count instead of a hardcoded 0.9, and pick the most-submitted /
highest-confidence candidate when several are returned. Adds httptest coverage
for the introdb client and provider.

Phase 1 of docs/superpowers/plans/2026-06-06-marker-sources-and-contribution-implementation.md

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(markers): multi-source dispatch, per-provider config, per-segment provenance

Add marker_provider_config (per-provider fetch enable/priority + contribute
gates, contribution off by default) and a cached ProviderConfigStore. Add
Registry.FetchMerged: query all fetch-enabled providers concurrently and keep
the best candidate per segment (submission_count, then confidence, then fetch
priority), stamping each winning marker with its provider/algorithm. Thread
per-segment provenance through MarkerUpdatePayload and scanner.MarkerUpdate
(additive SegmentProvenance overrides) so a merged result writes correct
per-segment provider/confidence/algorithm; the legacy shared columns keep a
summary. The lazy-playback path now uses FetchMerged. With only TheIntroDB
enabled, behavior is unchanged.

Phase 2 of docs/superpowers/plans/2026-06-06-marker-sources-and-contribution-implementation.md

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(markers): TheIntroDB submission client, contribution audit, service engine

Add a markers.Submitter capability and implement it on the introdb provider
(POST /v3/submit, GET /v3/user/stats; key required, usage-limit aware, applies
the null start/end conventions). Add the marker_contributions audit table and a
value-hash-keyed ContributionStore for idempotency. Add ContributionService:
resolves enabled submitter providers, gates eligibility (never re-submit
online-sourced markers; auto runs require contribute_auto_local + scanner-intro
above the per-provider confidence threshold), checks idempotency, submits, and
records. Wired in main.go; no trigger yet (admin API and task follow).

Phase 3 of docs/superpowers/plans/2026-06-06-marker-sources-and-contribution-implementation.md

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(api): admin marker editing, contribution, and provider config endpoints

Add the RequireAdmin marker API: GET/PUT /admin/files/{id}/markers (read with
provenance; manual upsert where a segment object sets and null clears),
DELETE .../markers/{segment}, POST .../contribute and GET .../contributions,
plus GET/PUT /admin/markers/providers[/{provider}] and a
.../validate key-check returning user stats. Manual writes go through the
priority-gated UpsertMarkers (source=manual) and notify live sessions; a new
FileRepository.ClearMarkers nulls a segment's columns. Validation mirrors the
contribution rules.

Phase 4 of docs/superpowers/plans/2026-06-06-marker-sources-and-contribution-implementation.md

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(markers): daily auto-contribution task for local intro markers

Add ContributeMarkersTask (daily 04:00, after local detection): when a provider
has contribute_enabled + contribute_auto_local, page through episode files with
a scanner intro marker at/above the provider's confidence threshold (new
ContributionStore.CandidateLocalIntroFiles keyset query) and run them through
ContributionService with Auto=true. No-op when no provider opts in; idempotent
and resumable across runs.

Phase 5 of docs/superpowers/plans/2026-06-06-marker-sources-and-contribution-implementation.md

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix(intromarkers): refine chromaprint starts with dialogue cues

* feat(markers): finish marker management backend

* feat(web): add marker editing UI

* feat(markers): use plugin marker providers

* fix(markers): address PR review feedback

* feat(player): show marker labels on seek hover

* fix(markers): type nullable marker mutation params

* feat(markers): audit marker edits and add permission

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-06 22:29:26 -04:00

256 lines
8.0 KiB
Go

package cache
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/redis/go-redis/v9"
)
// ---------------------------------------------------------------------------
// Channel constants
// ---------------------------------------------------------------------------
const (
// ChannelCatalog is the pub/sub channel for catalog events (scan_complete,
// metadata_updated).
ChannelCatalog = "silo:catalog"
// ChannelAdmin is the pub/sub channel for admin events (user_disabled,
// user_deleted, settings_changed).
ChannelAdmin = "silo:admin"
// ChannelPlayback is the pub/sub channel reserved for future playback
// events.
ChannelPlayback = "silo:playback"
// ChannelLogs is the pub/sub channel for persisted operational/audit log
// entries that should be fanned out to admin WebSocket subscribers.
ChannelLogs = "silo:logs"
// ChannelEvents is the pub/sub channel for passive websocket events.
ChannelEvents = "silo:events"
)
// ---------------------------------------------------------------------------
// Event type constants
// ---------------------------------------------------------------------------
const (
EventScanComplete = "scan_complete"
EventMetadataUpdated = "metadata_updated"
EventAdminStatsInvalidated = "admin_stats_invalidated"
EventPlaybackSessionsChanged = "playback_sessions_changed"
EventUserDisabled = "user_disabled"
EventUserDeleted = "user_deleted"
EventSettingsChanged = "settings_changed"
EventMarkerProviderConfigChanged = "marker_provider_config_changed"
EventNodePoolChanged = "node_pool_changed"
EventOperationalLogAppended = "operational_log_appended"
EventAuditLogAppended = "audit_log_appended"
EventEventsNotification = "events_notification"
)
// ---------------------------------------------------------------------------
// Event
// ---------------------------------------------------------------------------
// Event represents a cache invalidation event transmitted over pub/sub.
type Event struct {
Type string `json:"type"` // e.g. "user_disabled", "scan_complete"
Payload string `json:"payload"` // e.g. user ID, library ID
}
// EventHandler is a callback invoked when an event is received on a
// subscribed channel.
type EventHandler func(event Event)
// ---------------------------------------------------------------------------
// EventBus interface
// ---------------------------------------------------------------------------
// EventBus is the interface for publishing and subscribing to cache
// invalidation events. When Redis is not configured the factory returns
// a no-op implementation so callers never need nil checks.
type EventBus interface {
// Publish sends an event on the given channel.
Publish(ctx context.Context, channel string, event Event) error
// Subscribe registers a handler for events on the given channel.
// The handler is called in a dedicated goroutine.
Subscribe(ctx context.Context, channel string, handler EventHandler) error
// Close tears down the bus and stops all subscriber goroutines.
// It is safe to call Close multiple times.
Close() error
}
// NewEventBus returns an EventBus. When redisURL is empty a no-op
// implementation is returned that silently succeeds on every call and
// has zero external dependencies.
func NewEventBus(redisURL string) EventBus {
if redisURL == "" {
return &NoopEventBus{}
}
return newRedisEventBus(redisURL)
}
// ---------------------------------------------------------------------------
// NoopEventBus
// ---------------------------------------------------------------------------
// NoopEventBus is a silent no-op implementation of EventBus used when
// Redis is not configured. All methods return nil without doing anything.
type NoopEventBus struct{}
// Publish is a no-op that always returns nil.
func (n *NoopEventBus) Publish(_ context.Context, _ string, _ Event) error { return nil }
// Subscribe is a no-op that always returns nil. The handler is never called.
func (n *NoopEventBus) Subscribe(_ context.Context, _ string, _ EventHandler) error { return nil }
// Close is a no-op that always returns nil. Safe to call multiple times.
func (n *NoopEventBus) Close() error { return nil }
// ---------------------------------------------------------------------------
// RedisEventBus
// ---------------------------------------------------------------------------
// subscription tracks a single Redis pub/sub subscription so it can be
// torn down on Close.
type subscription struct {
pubsub *redis.PubSub
cancel context.CancelFunc
}
// RedisEventBus implements EventBus using Redis pub/sub. Events are
// JSON-serialized before being published.
type RedisEventBus struct {
client *redis.Client
mu sync.Mutex
subs []subscription
once sync.Once // ensures Close is idempotent
done chan struct{}
}
// newRedisEventBus creates a RedisEventBus connected to the given Redis URL.
func newRedisEventBus(redisURL string) *RedisEventBus {
opts, err := redis.ParseURL(redisURL)
if err != nil {
// If the URL cannot be parsed, treat it as a simple address.
opts = &redis.Options{Addr: redisURL}
}
return &RedisEventBus{
client: redis.NewClient(opts),
done: make(chan struct{}),
}
}
// Publish serializes the event as JSON and publishes it on the given
// Redis channel.
func (r *RedisEventBus) Publish(ctx context.Context, channel string, event Event) error {
data, err := json.Marshal(event)
if err != nil {
return err
}
return r.client.Publish(ctx, channel, data).Err()
}
// Subscribe creates a Redis pub/sub subscription on the given channel and
// spawns a goroutine that delivers incoming messages to the handler.
func (r *RedisEventBus) Subscribe(ctx context.Context, channel string, handler EventHandler) error {
subCtx, cancel := context.WithCancel(ctx)
pubsub := r.client.Subscribe(subCtx, channel)
// Wait for the subscription to be confirmed.
if _, err := pubsub.Receive(subCtx); err != nil {
cancel()
return err
}
r.mu.Lock()
r.subs = append(r.subs, subscription{pubsub: pubsub, cancel: cancel})
r.mu.Unlock()
go r.listen(pubsub, handler)
return nil
}
// listen reads messages from the pub/sub channel and invokes the handler.
func (r *RedisEventBus) listen(pubsub *redis.PubSub, handler EventHandler) {
ch := pubsub.Channel()
for {
select {
case msg, ok := <-ch:
if !ok {
return
}
var evt Event
if err := json.Unmarshal([]byte(msg.Payload), &evt); err != nil {
// Skip malformed messages.
continue
}
handler(evt)
case <-r.done:
return
}
}
}
// Close stops all subscriber goroutines and closes the Redis client.
// It is safe to call Close multiple times; only the first call has any
// effect.
func (r *RedisEventBus) Close() error {
var firstErr error
r.once.Do(func() {
close(r.done)
r.mu.Lock()
subs := r.subs
r.subs = nil
r.mu.Unlock()
for _, s := range subs {
if err := s.pubsub.Close(); err != nil && firstErr == nil {
firstErr = err
}
s.cancel()
}
if err := r.client.Close(); err != nil && firstErr == nil {
firstErr = err
}
})
return firstErr
}
// ---------------------------------------------------------------------------
// Redis client factory
// ---------------------------------------------------------------------------
// NewRedisClient creates a Redis client based on the config.
// Returns nil and no error if no Redis URL or Sentinel config is provided.
// Returns an error if configuration is present but invalid.
func NewRedisClient(cfg config.RedisConfig) (*redis.Client, error) {
if cfg.SentinelMaster != "" && len(cfg.SentinelAddresses) > 0 {
return redis.NewFailoverClient(&redis.FailoverOptions{
MasterName: cfg.SentinelMaster,
SentinelAddrs: cfg.SentinelAddresses,
SentinelPassword: cfg.SentinelPassword,
DB: 0,
}), nil
}
if cfg.URL == "" {
return nil, nil
}
opt, err := redis.ParseURL(cfg.URL)
if err != nil {
return nil, fmt.Errorf("invalid redis URL: %w", err)
}
return redis.NewClient(opt), nil
}